From a09e4e2eb93ca64b8beec4efac4c24e7dedc667d Mon Sep 17 00:00:00 2001 From: zwzhao Date: Thu, 29 May 2014 08:57:18 +0000 Subject: [PATCH] bug TDI-29527 fixed : Error executing MR job using a Hadoop custom cluster based on CDH5 git-svn-id: http://talendforge.org/svn/tos/branches/branch-5_5@118076 f6f1c999-d317-4740-80b0-e6d1abc6f99e --- .../version/custom/ECustomVersionGroup.java | 2 ++ .../version/custom/ECustomVersionType.java | 2 ++ .../version/custom/HadoopVersionDialog.java | 29 ++++++++++++++++++- .../designer/core/IDesignerCoreService.java | 2 ++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/ECustomVersionGroup.java b/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/ECustomVersionGroup.java index 4245b122d4..53b953c1ac 100644 --- a/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/ECustomVersionGroup.java +++ b/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/ECustomVersionGroup.java @@ -28,6 +28,8 @@ public enum ECustomVersionGroup { PIG_HBASE, PIG_HCATALOG, + + MAP_REDUCE, ALL; diff --git a/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/ECustomVersionType.java b/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/ECustomVersionType.java index 3fb31121fb..d8138384d0 100644 --- a/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/ECustomVersionType.java +++ b/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/ECustomVersionType.java @@ -21,6 +21,8 @@ public enum ECustomVersionType { PIG_HBASE("Pig for HBase", ECustomVersionGroup.PIG_HBASE), //$NON-NLS-1$ PIG_HCATALOG("Pig for Hcatalog", ECustomVersionGroup.PIG_HCATALOG), //$NON-NLS-1$ + + MAP_REDUCE("Map Reduce", ECustomVersionGroup.MAP_REDUCE), //$NON-NLS-1$ ALL("All", ECustomVersionGroup.ALL); //$NON-NLS-1$ diff --git a/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/HadoopVersionDialog.java b/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/HadoopVersionDialog.java index 0e88093cad..e145c6c722 100644 --- a/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/HadoopVersionDialog.java +++ b/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/version/custom/HadoopVersionDialog.java @@ -34,6 +34,7 @@ import org.talend.core.GlobalServiceRegister; import org.talend.core.hadoop.IHadoopService; import org.talend.core.hadoop.version.EHadoopDistributions; import org.talend.core.hadoop.version.EHadoopVersion4Drivers; +import org.talend.core.model.components.ComponentCategory; import org.talend.core.model.general.ModuleNeeded; import org.talend.core.model.process.IElementParameter; import org.talend.core.model.process.INode; @@ -99,7 +100,7 @@ public class HadoopVersionDialog extends TitleAreaDialog { protected void configureShell(Shell newShell) { super.configureShell(newShell); newShell.setText(Messages.getString("HadoopVersionDialog.title")); //$NON-NLS-1$ - newShell.setSize(580, 450); + newShell.setSize(700, 450); setHelpAvailable(false); } @@ -411,6 +412,8 @@ public class HadoopVersionDialog extends TitleAreaDialog { if (ECustomVersionType.PIG == type || ECustomVersionType.PIG_HBASE == type || ECustomVersionType.PIG_HCATALOG == type) { hadoopLibraries = getLibrariesForPig(type); + } else if (ECustomVersionType.MAP_REDUCE == type) { + hadoopLibraries = getLibrariesForMapReduce(type); } else { // fix for TDI-25676 HCATALOG and OOZIE should use the same jars as HDFS if (!commonGroupCalculated @@ -447,6 +450,30 @@ public class HadoopVersionDialog extends TitleAreaDialog { return libMap; } + + private Set getLibrariesForMapReduce(ECustomVersionType type) { + Set neededLibraries = new HashSet(); + INode node = CoreRuntimePlugin.getInstance().getDesignerCoreService() + .getRefrenceNode("tMRConfiguration", ComponentCategory.CATEGORY_4_MAPREDUCE.getName());//$NON-NLS-1$ + + IElementParameter elementParameter = node.getElementParameter("DISTRIBUTION");//$NON-NLS-1$ + if (elementParameter != null) { + elementParameter.setValue(distribution); + } + + elementParameter = node.getElementParameter("MR_VERSION");//$NON-NLS-1$ + if (elementParameter != null) { + elementParameter.setValue(version); + } + + List modulesNeeded = node.getModulesNeeded(); + for (ModuleNeeded module : modulesNeeded) { + if (module.isRequired(node.getElementParameters())) { + neededLibraries.add(module.getModuleName()); + } + } + return neededLibraries; + } private Set getLibrariesForPig(ECustomVersionType type) { Set neededLibraries = new HashSet(); diff --git a/org.talend.core.runtime/src/main/java/org/talend/designer/core/IDesignerCoreService.java b/org.talend.core.runtime/src/main/java/org/talend/designer/core/IDesignerCoreService.java index 38dab822cc..d05017dc3f 100644 --- a/org.talend.core.runtime/src/main/java/org/talend/designer/core/IDesignerCoreService.java +++ b/org.talend.core.runtime/src/main/java/org/talend/designer/core/IDesignerCoreService.java @@ -110,6 +110,8 @@ public interface IDesignerCoreService extends IService { public boolean isTalendEditor(IEditorPart activeEditor); public INode getRefrenceNode(String componentName); + + public INode getRefrenceNode(String componentName, String paletteType); public boolean executeUpdatesManager(List results, boolean onlySimpleShow);