fix(TUP-34769) Error fields for Rejects get dropped from tSalesforceOutput after Sync Columns. (#5061)

fix bug TUP-33497

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/utils/workbench/gef/SimpleHtmlFigure.java

@@ -113,6 +113,10 @@ public class SimpleHtmlFigure extends Figure {
         add(horizContainer);
     }
 
+    public void setText(final String text) {
+        setText(text, false);
+    }
+
     /**
      * Display some HTML text..
      *
@@ -120,7 +124,7 @@ public class SimpleHtmlFigure extends Figure {
      * @param isSysDefaultColor true if use system default font color
      */
     @SuppressWarnings("unchecked")
-    public void setText(final String text) {
+    public void setText(final String text, boolean isSysDefaultColor) {
 
         if (this.text.equals(text)) {
             // if the text is the same, there's nothing to change, so return.
@@ -138,14 +142,16 @@ public class SimpleHtmlFigure extends Figure {
         newHorizContainer();
 
         List<Color> colorStack = new ArrayList<Color>();
-        buildFigures(text, SWT.None, colorStack);
+        colorStack.add(ColorConstants.black);
+        // for some dark theme system like Ubuntu,if true use default system font color
+        buildFigures(text, SWT.None, colorStack, isSysDefaultColor);
 
         setPreferredSize(computePreferedSize());
 
         this.text = text;
     }
 
-    private void buildFigures(final String newText, final int fontCode, final List<Color> colorStack) {
+    private void buildFigures(final String newText, final int fontCode, final List<Color> colorStack, boolean isSysDefaultColor) {
         // Optimize
         if (newText == null || newText.length() == 0) {
             return;
@@ -161,7 +167,7 @@ public class SimpleHtmlFigure extends Figure {
         if (isFirstIndex(boldIndex, italicIndex, fontIndex, brIndex)) {
             if (boldIndex > 0) {
                 String begText = newText.substring(0, boldIndex);
-                buildFigures(begText, newFontCode, colorStack);
+                buildFigures(begText, newFontCode, colorStack, isSysDefaultColor);
             }
             newFontCode = newFontCode | SWT.BOLD;
 
@@ -170,17 +176,17 @@ public class SimpleHtmlFigure extends Figure {
             if (endBoldIndex != -1) {
                 String boldText = newText.substring(boldIndex + TAG_BOLD_BEG.length(), endBoldIndex);
                 endText = newText.substring(endBoldIndex + TAG_BOLD_END.length());
-                buildFigures(boldText, newFontCode, colorStack);
+                buildFigures(boldText, newFontCode, colorStack, isSysDefaultColor);
             } else {
                 endText = newText.substring(boldIndex + TAG_BOLD_BEG.length());
             }
 
             newFontCode = newFontCode ^ SWT.BOLD;
-            buildFigures(endText, newFontCode, colorStack);
+            buildFigures(endText, newFontCode, colorStack, isSysDefaultColor);
         } else if (isFirstIndex(italicIndex, boldIndex, fontIndex, brIndex)) {
             if (italicIndex > 0) {
                 String begText = newText.substring(0, italicIndex);
-                buildFigures(begText, newFontCode, colorStack);
+                buildFigures(begText, newFontCode, colorStack, isSysDefaultColor);
             }
             newFontCode = newFontCode | SWT.ITALIC;
 
@@ -189,59 +195,52 @@ public class SimpleHtmlFigure extends Figure {
             if (endItalicIndex != -1) {
                 String italicText = newText.substring(italicIndex + TAG_ITALIC_BEG.length(), endItalicIndex);
                 endText = newText.substring(endItalicIndex + TAG_ITALIC_END.length());
-                buildFigures(italicText, newFontCode, colorStack);
+                buildFigures(italicText, newFontCode, colorStack, isSysDefaultColor);
             } else {
                 endText = newText.substring(italicIndex + TAG_ITALIC_BEG.length());
             }
             newFontCode = newFontCode ^ SWT.ITALIC;
 
-            buildFigures(endText, newFontCode, colorStack);
+            buildFigures(endText, newFontCode, colorStack, isSysDefaultColor);
         } else if (isFirstIndex(fontIndex, boldIndex, italicIndex, brIndex)) {
             if (fontIndex > 0) {
                 String begText = newText.substring(0, fontIndex);
-                buildFigures(begText, newFontCode, colorStack);
+                buildFigures(begText, newFontCode, colorStack, isSysDefaultColor);
             }
             int colorIndex = newText.indexOf(TAG_FONT_COLOR_BEG_1);
 
-            Color color = null;
+            Color color;
             int colorIndex2 = newText.indexOf(TAG_FONT_BEG_2);
             if (colorIndex2 != -1) {
                 String colorCode = newText.substring(colorIndex + TAG_FONT_COLOR_BEG_1.length(), colorIndex2);
                 color = getColor(colorCode);
-            } else if (colorStack.size() > 0){
+            } else {
                 color = colorStack.get(colorStack.size() - 1);
             }
-            
-            boolean isPushed = false;
-            if (color != null) {
-                colorStack.add(color);
-                isPushed = true;
-            }
+            colorStack.add(color);
 
             String endText;
             int endColorIndex = newText.indexOf(TAG_FONT_END);
             if (endColorIndex != -1) {
                 String colorText = newText.substring(colorIndex2 + TAG_FONT_BEG_2.length(), endColorIndex);
                 endText = newText.substring(endColorIndex + TAG_FONT_END.length());
-                buildFigures(colorText, newFontCode, colorStack);
+                buildFigures(colorText, newFontCode, colorStack, isSysDefaultColor);
             } else {
                 endText = newText.substring(colorIndex2 + TAG_FONT_BEG_2.length());
             }
 
-            if (isPushed) {
-                colorStack.remove(colorStack.size() - 1); 
-            }
-            buildFigures(endText, newFontCode, colorStack);
+            colorStack.remove(colorStack.size() - 1);
+            buildFigures(endText, newFontCode, colorStack, isSysDefaultColor);
         } else if (isFirstIndex(brIndex, boldIndex, italicIndex, fontIndex)) {
             if (brIndex > 0) {
                 String begText = newText.substring(0, brIndex);
-                buildFigures(begText, newFontCode, colorStack);
+                buildFigures(begText, newFontCode, colorStack, isSysDefaultColor);
             }
 
             newHorizContainer();
 
             String endText = newText.substring(brIndex + TAG_BR.length());
-            buildFigures(endText, newFontCode, colorStack);
+            buildFigures(endText, newFontCode, colorStack, isSysDefaultColor);
         } else {
             Font fontToUse;
             Label label = new Label();
@@ -260,7 +259,7 @@ public class SimpleHtmlFigure extends Figure {
                 }
             }
             label.setFont(fontToUse);
-            if (colorStack.size() > 0) {
+            if (!isSysDefaultColor) {
                 label.setForegroundColor(colorStack.get(colorStack.size() - 1));
             }
             horizContainer.add(label);

main/plugins/org.talend.core.repository/plugin.xml

@@ -93,8 +93,8 @@
        <RepositoryComponent
              name="EXASOL"
              withSchema="true"
-             input="tExasolInput"
-             output="tExasolOutput">
+             input="tEXAInput"
+             output="tEXAOutput">
           <Item
                 clazz="org.talend.core.model.properties.DatabaseConnectionItem">
           </Item>

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/HadoopConstants.java

@@ -26,6 +26,8 @@ public class HadoopConstants {
     public static final String SPARK_MODE_YARN_CLIENT = "YARN_CLIENT";
 
     public static final String SPARK_MODE_YARN_CLUSTER = "YARN_CLUSTER";
+    
+    public static final String SPARK_MODE_DATAPROC = "DATAPROC";
 
     public static final String FRAMEWORK = "FRAMEWORK";
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/IMetadataTable.java

@@ -71,6 +71,8 @@ public interface IMetadataTable {
     
     public boolean sameMetadataAs(IMetadataTable other, int options, boolean order);
 
+    public boolean sameMetadataAs(List<IMetadataColumn> otherMetadataTableColumns, int options, boolean order);
+
     public void sortCustomColumns();
 
     public boolean isReadOnly();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataTable.java

@@ -257,8 +257,13 @@ public class MetadataTable implements IMetadataTable, Cloneable {
         if (!(input instanceof IMetadataTable)) {
             return false;
         }
-        List<IMetadataColumn> thisColumnListWithUnselected = this.getListColumns(true);
         List<IMetadataColumn> inputColumnListWithUnselected = input.getListColumns(true);
+        return sameMetadataAs(inputColumnListWithUnselected, options, order);
+    }
+
+    @Override
+    public boolean sameMetadataAs(List<IMetadataColumn> inputColumnListWithUnselected, int options, boolean order) {
+        List<IMetadataColumn> thisColumnListWithUnselected = this.getListColumns(true);
         if (thisColumnListWithUnselected == null) {
             if (inputColumnListWithUnselected != null) {
                 return false;
@@ -300,9 +305,6 @@ public class MetadataTable implements IMetadataTable, Cloneable {
                 // no matter if this one is custom or not (all custom must be propagated too)
                 for (int i = 0; i < inputColumnListWithUnselected.size(); i++) {
                     IMetadataColumn inputColumn = inputColumnListWithUnselected.get(i);
-                    if(inputColumn.isCustom()) {
-                        continue;
-                    }
                     IMetadataColumn myColumn = this.getColumn(inputColumn.getLabel());
                     outputColumnsNotTested.remove(myColumn);
                     if (!inputColumn.sameMetacolumnAs(myColumn, options)) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ProcessUtils.java

@@ -1086,6 +1086,18 @@ public final class ProcessUtils {
                     }
                 }
             }
+            if ("cHttp".equals(node.getComponentName())) {
+                for (Object elementParameter : node.getElementParameter()) {
+                    ElementParameterType elementParameterType = (ElementParameterType)elementParameter;
+
+                    String name = elementParameterType.getName();
+                    String value = elementParameterType.getValue();
+
+                    if ("SERVER".equals(name) && (value != null && "true".equals(value.toString()))) {
+                        return true;
+                    }
+                }
+            }
         }
 
         return false;

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenJobPom.java

@@ -674,11 +674,82 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         Set<String> talendLibCoordinate = new HashSet<>();
         Set<String> _3rdLibCoordinate = new HashSet<>();
         Map<String, Set<Dependency>> duplicateLibs = new HashMap<>();
+        Set<Dependency> parentJobDependencies = new HashSet<Dependency>();
         IProcessor processor = getJobProcessor();
+        Map<String, Object> processorArgs = processor.getArguments();
+        
+        boolean buidTypeAsMs = false;
+        boolean needLauncher = false;
+        for (Map.Entry<String, Object> entry : processorArgs.entrySet()) {
+            String key = entry.getKey();
+            Object val = entry.getValue();
+            if (!buidTypeAsMs && TalendProcessArgumentConstant.ARG_BUILD_TYPE.equals(key)
+                    && ("ROUTE_MICROSERVICE".equals(val) || "REST_MS".equals(val))) {
+                buidTypeAsMs = true;
+            }
+            if (!needLauncher && TalendProcessArgumentConstant.ARG_NEED_LAUNCHER.equals(key)
+                    && Boolean.TRUE.equals(val)) {
+                needLauncher = true;
+            }
+        }
+        
+        boolean isBuildAsMsZip = buidTypeAsMs && needLauncher;
 
-        // current job
+        // talend libraries and codes
+        List<Dependency> dependencies = new ArrayList<>();
+        Set<ModuleNeeded> modules = new HashSet<>();
+         // current job
         Property currentJobProperty = processor.getProperty();
         jobCoordinate.add(getJobCoordinate(currentJobProperty));
+        if(!isBuildAsMsZip) {
+            // codes
+            List<Dependency> codeDependencies = new ArrayList<>();
+            addCodesDependencies(codeDependencies);
+            // codesjar
+            codeDependencies.addAll(getCodesJarDependenciesFromChildren());
+            dependencies.addAll(codeDependencies);
+
+            // codes dependencies (optional)
+            ERepositoryObjectType.getAllTypesOfCodes().forEach(t -> dependencies.addAll(PomUtil.getCodesDependencies(t)));
+            
+            // libraries of talend/3rd party
+            parentJobDependencies  = processor
+                    .getNeededModules(
+                            TalendProcessOptionConstants.MODULES_EXCLUDE_SHADED | TalendProcessOptionConstants.MODULES_WITH_CODESJAR)
+                    .stream()
+                    .filter(m -> !m.isExcluded()).map(m -> createDenpendency(m, false))
+                    .collect(Collectors.toSet());
+            dependencies.addAll(parentJobDependencies);
+            
+            // get codesjar libraries from related joblets
+            dependencies.addAll(processor.getCodesJarModulesNeededOfJoblets().stream().map(m -> createDenpendency(m, false))
+                    .collect(Collectors.toSet()));
+            
+            // testcase modules from current job (optional)
+            modules.addAll(ProcessorDependenciesManager.getTestcaseNeededModules(currentJobProperty));
+            dependencies.addAll(
+                    modules.stream().filter(m -> !m.isExcluded()).map(m -> createDenpendency(m, true)).collect(Collectors.toSet()));
+
+            dependencies.stream().filter(d -> !MavenConstants.PACKAGING_POM.equals(d.getType())).forEach(d -> {
+                String coordinate = getCoordinate(d);
+                String groupId = d.getGroupId();
+                boolean optional = ((SortableDependency) d).isAssemblyOptional();
+                if (jobCoordinate.contains(coordinate) || talendLibCoordinate.contains(coordinate)
+                        || _3rdLibCoordinate.contains(coordinate)) {
+                    return;
+                }
+                if (MavenConstants.DEFAULT_LIB_GROUP_ID.equals(groupId) || codeDependencies.contains(d)) {
+                    if (!optional) {
+                        talendLibCoordinate.add(coordinate);
+                    }
+                } else {
+                    if (!optional) {
+                        _3rdLibCoordinate.add(coordinate);
+                    }
+                    addToDuplicateLibs(duplicateLibs, d);
+                }
+            });
+        }
         
         // children jobs without test cases
         Set<JobInfo> childrenJobInfo = processor.getBuildChildrenJobs().stream().filter(j -> !j.isTestContainer())
@@ -687,31 +758,6 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
             childrenJobInfo.forEach(j -> jobCoordinate.add(getJobCoordinate(j.getProcessItem().getProperty())));
         }
 
-        // talend libraries and codes
-        List<Dependency> dependencies = new ArrayList<>();
-        // codes
-        List<Dependency> codeDependencies = new ArrayList<>();
-        addCodesDependencies(codeDependencies);
-        // codesjar
-        codeDependencies.addAll(getCodesJarDependenciesFromChildren());
-        dependencies.addAll(codeDependencies);
-
-        // codes dependencies (optional)
-        ERepositoryObjectType.getAllTypesOfCodes().forEach(t -> dependencies.addAll(PomUtil.getCodesDependencies(t)));
-
-        // libraries of talend/3rd party
-        Set<Dependency> parentJobDependencies = processor
-                .getNeededModules(
-                        TalendProcessOptionConstants.MODULES_EXCLUDE_SHADED | TalendProcessOptionConstants.MODULES_WITH_CODESJAR)
-                .stream()
-                .filter(m -> !m.isExcluded()).map(m -> createDenpendency(m, false))
-                .collect(Collectors.toSet());
-        dependencies.addAll(parentJobDependencies);
-
-        // get codesjar libraries from related joblets
-        dependencies.addAll(processor.getCodesJarModulesNeededOfJoblets().stream().map(m -> createDenpendency(m, false))
-                .collect(Collectors.toSet()));
-
         // missing modules from the job generation of children
         Map<String, Set<Dependency>> childjobDependencies = new HashMap<String, Set<Dependency>>();
         childrenJobInfo.forEach(j -> {
@@ -720,40 +766,15 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
                             LastGenerationInfo.getInstance().getCodesJarModulesNeededPerJob(j.getJobId(), j.getJobVersion())
                                     .stream())
                     .filter(m -> !m.isExcluded()).map(m -> createDenpendency(m, false)).collect(Collectors.toSet());
-            dependencies.addAll(collectDependency);
+            if(!isBuildAsMsZip) {
+                dependencies.addAll(collectDependency);
+            }
             childjobDependencies.put(j.getJobId(), collectDependency);});
-
-        Set<ModuleNeeded> modules = new HashSet<>();
-        // testcase modules from current job (optional)
-        modules.addAll(ProcessorDependenciesManager.getTestcaseNeededModules(currentJobProperty));
-
+     
         // testcase modules from children job (optional)
         childrenJobInfo.forEach(
                 j -> modules.addAll(ProcessorDependenciesManager.getTestcaseNeededModules(j.getProcessItem().getProperty())));
 
-        dependencies.addAll(
-                modules.stream().filter(m -> !m.isExcluded()).map(m -> createDenpendency(m, true)).collect(Collectors.toSet()));
-
-        dependencies.stream().filter(d -> !MavenConstants.PACKAGING_POM.equals(d.getType())).forEach(d -> {
-            String coordinate = getCoordinate(d);
-            String groupId = d.getGroupId();
-            boolean optional = ((SortableDependency) d).isAssemblyOptional();
-            if (jobCoordinate.contains(coordinate) || talendLibCoordinate.contains(coordinate)
-                    || _3rdLibCoordinate.contains(coordinate)) {
-                return;
-            }
-            if (MavenConstants.DEFAULT_LIB_GROUP_ID.equals(groupId) || codeDependencies.contains(d)) {
-                if (!optional) {
-                    talendLibCoordinate.add(coordinate);
-                }
-            } else {
-                if (!optional) {
-                    _3rdLibCoordinate.add(coordinate);
-                }
-                addToDuplicateLibs(duplicateLibs, d);
-            }
-        });
-
         Iterator<String> iterator = duplicateLibs.keySet().iterator();
         while (iterator.hasNext()) {
             Set<Dependency> dupDependencies = duplicateLibs.get(iterator.next());
@@ -772,25 +793,28 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
 
         try {
             Document document = PomUtil.loadAssemblyFile(null, assemblyFile);
-            // add talend libs & codes
-            setupDependencySetNode(document, talendLibCoordinate, "lib", "${artifact.artifactId}.${artifact.extension}", false,
-                    false);
-            // add 3rd party libs: groupId:artifactId:type:version
-            setupDependencySetNode(document,
-                    _3rdLibCoordinate.stream().filter(s -> s.split(":").length == 4).collect(Collectors.toSet()), "lib", null,
-                    false, false);
-            // add 3rd party libs with classifier: groupId:artifactId:type:classifier:version
-            setupDependencySetNode(document,
-                    _3rdLibCoordinate.stream().filter(s -> s.split(":").length == 5).collect(Collectors.toSet()), "lib", null,
-                    false, false);
-            // FIXME if later add classifier for org.talend.libraries libs, code and job artifact, need to handle it
-            // like 3rd libs as well
+            if(!isBuildAsMsZip) {
+             // add talend libs & codes
+                setupDependencySetNode(document, talendLibCoordinate, "lib", "${artifact.artifactId}.${artifact.extension}", false,
+                        false, false);
+                // add 3rd party libs: groupId:artifactId:type:version
+                setupDependencySetNode(document,
+                        _3rdLibCoordinate.stream().filter(s -> s.split(":").length == 4).collect(Collectors.toSet()), "lib", null,
+                        false, false, false);
+                // add 3rd party libs with classifier: groupId:artifactId:type:classifier:version
+                setupDependencySetNode(document,
+                        _3rdLibCoordinate.stream().filter(s -> s.split(":").length == 5).collect(Collectors.toSet()), "lib", null,
+                        false, false, false);
+                // FIXME if later add classifier for org.talend.libraries libs, code and job artifact, need to handle it
+                // like 3rd libs as well
+                
+                // add duplicate dependencies if exists
+                setupFileNode(document, parentJobDependencies, childjobDependencies, duplicateLibs);
+            }
 
             // add jobs
             setupDependencySetNode(document, jobCoordinate, "${talend.job.name}",
-                    "${artifact.build.finalName}.${artifact.extension}", true, false);
-            // add duplicate dependencies if exists
-            setupFileNode(document, parentJobDependencies, childjobDependencies, duplicateLibs);
+                    "${artifact.build.finalName}.${artifact.extension}", true, false, isBuildAsMsZip);
 
             PomUtil.saveAssemblyFile(assemblyFile, document);
         } catch (Exception e) {
@@ -890,12 +914,12 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
     }
 
     protected void setupDependencySetNode(Document document, Set<String> libIncludes, String outputDir, String fileNameMapping,
-            boolean useProjectArtifact, boolean unpack) {
-        if (libIncludes.isEmpty()) {
+            boolean useProjectArtifact, boolean unpack, boolean isBuildAsMsZip) {
+        if (!isBuildAsMsZip && libIncludes.isEmpty()) {
             return;
         }
         Node dependencySetsNode = document.getElementsByTagName("dependencySets").item(0);
-        if (dependencySetsNode == null) {
+        if (!isBuildAsMsZip && dependencySetsNode == null) {
             return;
         }
         Node dependencySetNode = document.createElement("dependencySet");

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenStandardJobOSGiPom.java

@@ -194,7 +194,7 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
             Document document = PomUtil.loadAssemblyFile(null, assemblyFile);
             // add jobs
             setupDependencySetNode(document, jobCoordinate, null, "${artifact.build.finalName}.${artifact.extension}", true,
-                    true);
+                    true, false);
             PomUtil.saveAssemblyFile(assemblyFile, document);
         } catch (Exception e) {
             ExceptionHandler.process(e);

main/plugins/org.talend.librariesmanager/resources/java/routines/system/ParserUtils.java

@@ -313,10 +313,14 @@ public class ParserUtils {
      */
     public static java.util.Date parseTo_Date(String dateString, String pattern) {
         // check the parameter for supporting " ","2007-09-13"," 2007-09-13 "
+        if (dateString != null) {
+            dateString = dateString.trim();
+        }
+
         if (dateString == null || dateString.length() == 0) {
             return null;
         }
-        dateString = dateString.trim();
+
         if (pattern == null) {
             pattern = Constant.dateDefaultPattern;
         }