fix(TUP-30849):Improve build Job performance (#4151)

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849

Signed-off-by: jding-tlnd <jding@talend.com>

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849
Signed-off-by: jding-tlnd <jding@talend.com>

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849

Signed-off-by: jding-tlnd <jding@talend.com>

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849

Signed-off-by: jding-tlnd <jding@talend.com>

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849

Signed-off-by: jding-tlnd <jding@talend.com>

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849

Signed-off-by: jding-tlnd <jding@talend.com>

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849

Signed-off-by: jding-tlnd <jding@talend.com>

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849

* feat(TUP-31117):Improve performances related to recursive jobs (#4290)

* feat(TUP-31117):Improve performances related to recursive jobs
https://jira.talendforge.org/browse/TUP-31117

* feat(TUP-31117):Improve performances related to recursive jobs
https://jira.talendforge.org/browse/TUP-31117

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/AbstractEMFRepositoryFactory.java

@@ -12,11 +12,14 @@
 // ============================================================================
 package org.talend.core.repository.model;
 
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.net.URL;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -623,8 +626,7 @@ public abstract class AbstractEMFRepositoryFactory extends AbstractRepositoryFac
             stream.close();
 
             byte[] currentContent = item.getContent().getInnerContent();
-
-            if (!Arrays.equals(innerContent, currentContent)) {
+            if (!isSameStringContent(innerContent, currentContent)) {
                 item.getContent().setInnerContent(innerContent);
                 Project project = getRepositoryContext().getProject();
                 save(project, item);
@@ -641,6 +643,44 @@ public abstract class AbstractEMFRepositoryFactory extends AbstractRepositoryFac
             throw new PersistenceException(ioe);
         }
     }
+    
+    protected boolean isSameStringContent(byte[] data1, byte[] data2) throws IOException {
+        boolean isSame = true;
+        BufferedReader br1 = null, br2 = null;
+        try {
+            br1 = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(data1), StandardCharsets.UTF_8.toString()));
+            br2 = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(data2), StandardCharsets.UTF_8.toString()));
+            String line1 = null, line2 = null;
+            while (isSame) {
+                line1 = br1.readLine();
+                line2 = br2.readLine();
+                if ((line1 == null && line2 == null)) {
+                    break;
+                }
+                if (!StringUtils.equals(line1, line2)) {
+                    isSame = false;
+                    break;
+                }
+            }
+        } finally {
+            if (br1 != null) {
+                try {
+                    br1.close();
+                } catch (IOException e) {
+                    ExceptionHandler.process(e);
+                }
+            }
+            if (br2 != null) {
+                try {
+                    br2.close();
+                } catch (IOException e) {
+                    ExceptionHandler.process(e);
+                }
+            }
+        }
+
+        return isSame;
+    }
 
     private void createSQLPattern(URL url, String sqlPatternLabel, String categoryName) throws PersistenceException {
         if (url == null) {
@@ -693,8 +733,7 @@ public abstract class AbstractEMFRepositoryFactory extends AbstractRepositoryFac
             stream.close();
 
             byte[] currentContent = item.getContent().getInnerContent();
-
-            if (!Arrays.equals(innerContent, currentContent)) {
+            if (!isSameStringContent(innerContent, currentContent)) {
                 item.getContent().setInnerContent(innerContent);
                 Project project = getRepositoryContext().getProject();
                 save(project, item);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/EDatabaseTypeName.java

@@ -114,7 +114,10 @@ public enum EDatabaseTypeName {
             "ParAccel", "ParAccel", Boolean.TRUE, "PARACCEL", "PARACCEL", EDatabaseSchemaOrCatalogMapping.Sid, EDatabaseSchemaOrCatalogMapping.Schema), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
     NETEZZA(
             "Netezza", "Netezza", Boolean.FALSE, "NETEZZA", EDatabaseSchemaOrCatalogMapping.Sid, EDatabaseSchemaOrCatalogMapping.None), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-    SAS("SAS", "SAS", Boolean.TRUE, "SAS", EDatabaseSchemaOrCatalogMapping.None, EDatabaseSchemaOrCatalogMapping.Schema), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+    /**
+     * @deprecated 
+     */
+    SAS("SAS", "SAS (Deprecated)", Boolean.TRUE, "SAS", EDatabaseSchemaOrCatalogMapping.None, EDatabaseSchemaOrCatalogMapping.Schema), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
     // General JDBC not support schema defalut
     GENERAL_JDBC(
             "General JDBC", "General JDBC", Boolean.FALSE, "JDBC", EDatabaseSchemaOrCatalogMapping.Sid, EDatabaseSchemaOrCatalogMapping.None), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/EImpalaDriver.java

@@ -22,14 +22,14 @@ import org.talend.core.runtime.hd.hive.HiveMetadataHelper;
  */
 public enum EImpalaDriver {
 
-    HIVE2("HIVE2", "HIVE2", "org.apache.hive.jdbc.HiveDriver"),
-    IMPALA40("IMPALA40", "IMPALA40", "com.cloudera.impala.jdbc4.Driver"),
-    IMPALA41("IMPALA41", "IMPALA41", "com.cloudera.impala.jdbc41.Driver");
+    HIVE2("HIVE2", "HIVE2", "org.apache.hive.jdbc.HiveDriver", "doSupportHive2"),
+    IMPALA("IMPALA", "IMPALA", "com.cloudera.impala.jdbc.Driver", "doSupportImpalaConnector");
 
-    EImpalaDriver(String displayName, String name, String driver) {
+    EImpalaDriver(String displayName, String name, String driver, String supportDriverMethodName) {
         this.displayName = displayName;
         this.name = name;
         this.driver = driver;
+        this.supportDriverMethodName = supportDriverMethodName;
     }
 
     private String displayName;
@@ -38,6 +38,8 @@ public enum EImpalaDriver {
 
     private String driver;
 
+    private String supportDriverMethodName;
+
     public String getDisplayName() {
         return displayName;
     }
@@ -50,6 +52,10 @@ public enum EImpalaDriver {
         return driver;
     }
 
+    public String getSupportDriverMethodName() {
+        return supportDriverMethodName;
+    }
+
     public static boolean isSupport(String distribution, String version, boolean byDisplay, String supportMethodName) {
         return HiveMetadataHelper.doSupportMethod(distribution, version, byDisplay, supportMethodName);
     }
@@ -57,7 +63,7 @@ public enum EImpalaDriver {
     public static String[] getImpalaDriverDisplay(String distribution, String version, boolean byDisplay) {
         List<String> list = new ArrayList<>(0);
         for (EImpalaDriver driver : EImpalaDriver.values()) {
-            if (isSupport(distribution, version, byDisplay, "doSupportImpalaConnector")) {
+            if (isSupport(distribution, version, byDisplay, driver.getSupportDriverMethodName())) {
                 list.add(driver.getDisplayName());
             }
         }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/DatabaseConnStrUtil.java

@@ -228,8 +228,10 @@ public class DatabaseConnStrUtil {
                     .get(ConnParameterKeys.IMPALA_AUTHENTICATION_PRINCIPLA));
         }
         String url = null;
-        if (template.startsWith(DbConnStrForHive.URL_HIVE_2_TEMPLATE)) {
-            url = getImpalaURLString(false, server, port, sidOrDatabase, impalaPrincipal);
+        String driver = dbConn.getParameters().get(ConnParameterKeys.IMPALA_DRIVER);
+        if (template.startsWith(DbConnStrForHive.URL_HIVE_2_TEMPLATE)
+                || template.startsWith(DbConnStrForHive.URL_IMPALA_TEMPLATE)) {
+            url = getImpalaURLString(false, server, port, sidOrDatabase, impalaPrincipal, driver);
         }
         url = attachAdditionalHiveParameters(url, dbConn, false);
         return url;
@@ -376,9 +378,13 @@ public class DatabaseConnStrUtil {
         return s;
     }
 
-    private static String getImpalaURLString(boolean supportContext, String server, String port, String sid, String Principal) {
-        String s = EDatabaseConnTemplate.IMPALA.getUrlTemplate(null);
-        String standardURlString = getImpalaURlString(s, supportContext, server, port, sid);
+    private static String getImpalaURLString(boolean supportContext, String server, String port, String sid, String Principal,
+            String driver) {
+        String template = EDatabaseConnTemplate.IMPALA.getUrlTemplate(null);
+        if (EDatabaseTypeName.IMPALA.getDbType().equals(driver)) {
+            template = EDatabaseConnTemplate.IMPALA_IMPALA_DRIVER.getUrlTemplate(null);
+        }
+        String standardURlString = getImpalaURlString(template, supportContext, server, port, sid);
         String principalSuffix = "principal="; //$NON-NLS-1$
         boolean hasPrinc = false;
         String[] urlArray = standardURlString.split(SEMICOLON);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/template/DbConnStrForHive.java

@@ -30,6 +30,8 @@ public class DbConnStrForHive extends DbConnStr {
 
     public static final String URL_HIVE_2_TEMPLATE = "jdbc:hive2://";//$NON-NLS-1$
 
+    public static final String URL_IMPALA_TEMPLATE = "jdbc:impala://";//$NON-NLS-1$
+
     /**
      * DOC Marvin DbConnStrForHive constructor comment.
      *

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/template/EDatabaseConnTemplate.java

@@ -214,6 +214,9 @@ public enum EDatabaseConnTemplate {
     IMPALA(new DbConnStr(EDatabaseTypeName.IMPALA, "jdbc:hive2://<host>:<port>/<sid>;auth=noSasl", //$NON-NLS-1$
             "21050", null, null, "localhost", "default")), //$NON-NLS-1$
 
+    IMPALA_IMPALA_DRIVER(new DbConnStr(EDatabaseTypeName.IMPALA, "jdbc:impala://<host>:<port>/<sid>;auth=noSasl", //$NON-NLS-1$
+            "21050", null, null, "localhost", "default")), //$NON-NLS-1$
+
     HBASE(new DbConnStr(EDatabaseTypeName.HBASE, "127.0.0.1", //$NON-NLS-1$
             "2181")), //$NON-NLS-1$
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/ISAPConstant.java

@@ -106,4 +106,6 @@ public interface ISAPConstant {
     public static final String PROP_DB_USERNAME = "db.username";//$NON-NLS-1$
 
     public static final String PROP_DB_PASSWORD = "db.password";//$NON-NLS-1$
-}
+
+    public static final String PROP_DB_ADDITIONAL_PROPERTIES = "db.additionalProperties";//$NON-NLS-1$
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolHelper.java

@@ -281,9 +281,7 @@ public final class MetadataToolHelper {
     }
 
     public static boolean isAllowSpecificCharacters() {
-        IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        return coreUIPluginNode
-                .getBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
+        return CoreRuntimePlugin.getInstance().getProjectPreferenceManager().isAllowSpecificCharacters();
     }
 
     /**

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/designerproperties/RepositoryToComponentProperty.java

@@ -443,6 +443,13 @@ public class RepositoryToComponentProperty {
             } else {
                 return TalendQuoteUtils.addQuotes(connection.getValue(dbPassword, false));
             }
+        } else if ("SAPHANA_PROPERTIES_STRING".equals(value)) { //$NON-NLS-1$
+            String dbParameters = TaggedValueHelper.getValueString(ISAPConstant.PROP_DB_ADDITIONAL_PROPERTIES, connection);
+            if (isContextMode(connection, dbParameters)) {
+                return dbParameters;
+            } else {
+                return TalendQuoteUtils.addQuotes(dbParameters);
+            }
         }
         return null;
     }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/IProcess.java

@@ -12,6 +12,7 @@
 // ============================================================================
 package org.talend.core.model.process;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
@@ -133,4 +134,8 @@ public interface IProcess extends IElement {
     public String getComponentsType();
 
     public INode getNodeByUniqueName(String uniqueName);
+
+    default List<? extends INode> getProcessNodes() {
+        return new ArrayList<INode>();
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/JobInfo.java

@@ -67,6 +67,8 @@ public class JobInfo {
 
     private boolean needUnloadProcessor;
 
+    private int includeESBFlag = 0;
+
     public JobInfo(String jobId, String contextName, String version) {
         this.jobId = jobId;
         this.contextName = contextName;
@@ -572,4 +574,13 @@ public class JobInfo {
     public boolean isNeedUnloadProcessor() {
         return this.needUnloadProcessor;
     }
+
+    public int getIncludeESBFlag() {
+        return includeESBFlag;
+    }
+
+    public void setIncludeESBFlag(int includeESBFlag) {
+        this.includeESBFlag = includeESBFlag;
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/ContextParameterUtils.java

@@ -36,6 +36,7 @@ import org.apache.oro.text.regex.Perl5Compiler;
 import org.apache.oro.text.regex.Perl5Matcher;
 import org.apache.oro.text.regex.Perl5Substitution;
 import org.apache.oro.text.regex.Util;
+import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.preferences.IEclipsePreferences;
 import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.talend.commons.utils.PasswordEncryptUtil;
@@ -52,6 +53,7 @@ import org.talend.core.model.process.IContextManager;
 import org.talend.core.model.process.IContextParameter;
 import org.talend.core.model.repository.IRepositoryPrefConstants;
 import org.talend.core.prefs.ITalendCorePrefConstants;
+import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.runtime.services.IGenericDBService;
 import org.talend.core.utils.TalendQuoteUtils;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextParameterType;
@@ -617,8 +619,12 @@ public final class ContextParameterUtils {
     }
 
     private static boolean isAllowSpecificCharacters() {
-        IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        return coreUIPluginNode.getBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
+        if (Platform.isRunning()) {
+            return CoreRuntimePlugin.getInstance().getProjectPreferenceManager().isAllowSpecificCharacters();
+        } else {
+            IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
+            return coreUIPluginNode.getBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
+        }
     }
 
     public static boolean isEmptyParameter(String source) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/NodeUtil.java

@@ -418,6 +418,52 @@ public class NodeUtil {
         return conns;
     }
 
+    /**
+     * DOC
+     * <p>
+     * The method searches for the incoming node connections of type
+     * on a processing path and returns the first ones only
+     * </p>
+     *
+     * @param node
+     * @param type - node type to look for
+     * @return
+     */
+    public static List<? extends IConnection> getFirstIncomingLineConnectionsOfType(INode node, String type) {
+        if (type == null)
+            return new ArrayList<IConnection>();
+
+        Set<String> uniqueNamesDone = new HashSet<String>();
+        List<? extends IConnection> allIncomingConnections = getFirstIncomingLineConnectionsOfType(node, uniqueNamesDone, type);
+
+        return allIncomingConnections;
+    }
+
+    private static List<? extends IConnection> getFirstIncomingLineConnectionsOfType(INode node, Set<String> uniqueNamesDone, String type) {
+        List<IConnection> conns = new ArrayList<IConnection>();
+
+        List<? extends IConnection> incomingConnections = node.getIncomingConnections();
+        if (incomingConnections != null) {
+
+            for (int i = 0; i < incomingConnections.size(); i++) {
+
+                IConnection connection = incomingConnections.get(i);
+                INode nextNode = connection.getSource();
+
+                if (!uniqueNamesDone.contains(nextNode.getUniqueName())) {
+                    uniqueNamesDone.add(nextNode.getUniqueName());
+
+                    if (type.equals((String)nextNode.getElementParameter("COMPONENT_NAME").getValue())) {
+                        conns.add(connection);
+                    } else {
+                        conns.addAll(getFirstIncomingLineConnectionsOfType(nextNode, uniqueNamesDone, type)); // follow this way
+                    }
+                }
+            }
+        }
+        return conns;
+    }
+
     public static INode getFirstMergeNode(INode node) {
         INode mergeNode = null;
         for (IConnection connection : node.getOutgoingConnections()) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/evaluator/AbstractPropertyValueEvaluator.java

@@ -126,6 +126,9 @@ public abstract class AbstractPropertyValueEvaluator implements PropertyValueEva
         }
 
         if (GenericTypeUtils.isStringType(property)) {
+            if (property.isFlag(Property.Flags.ENCRYPT)) {
+                return TalendQuoteUtils.removeQuotes(stringValue);
+            }
             return TalendQuoteUtils.removeQuotes(StringEscapeUtils.unescapeJava(stringValue));
         }
         return rawValue;

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenConstants.java

@@ -46,6 +46,8 @@ public interface MavenConstants {
 
     static final String EXCLUDE_DELETED_ITEMS = "EXCLUDE_DELETED_ITEMS";
 
+    static final String SKIP_LOOP_DEPENDENCY_CHECK = "SKIP_LOOP_DEPENDENCY_CHECK";
+
     static final String SKIP_FOLDERS = "SKIP_FOLDERS";
 
     /*

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/process/TalendProcessOptionConstants.java

@@ -80,4 +80,13 @@ public interface TalendProcessOptionConstants {
      */
     public static final int MODULES_WITH_CODESJAR = 1 << 6;
 
+    /**
+     * flag for check if is ESB job
+     */
+    public static final int ISESB_CHECKED = 1;
+
+    public static final int ISESB_CHILDREN_INCLUDE = 1 << 1;
+
+    public static final int ISESB_CURRENT_INCLUDE = 1 << 2;
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/projectsetting/ProjectPreferenceManager.java

@@ -15,6 +15,7 @@ package org.talend.core.runtime.projectsetting;
 import java.beans.PropertyChangeEvent;
 import java.beans.PropertyChangeListener;
 
+import org.apache.commons.lang3.StringUtils;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.resources.ProjectScope;
 import org.eclipse.core.runtime.Assert;
@@ -27,6 +28,7 @@ import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.model.general.Project;
+import org.talend.core.model.repository.IRepositoryPrefConstants;
 import org.talend.designer.runprocess.IRunProcessService;
 import org.talend.repository.ProjectManager;
 import org.talend.repository.documentation.ERepositoryActionName;
@@ -195,7 +197,19 @@ public class ProjectPreferenceManager {
         // return qulifierPreference.getBoolean(key, false);
         return getPreferenceStore().getBoolean(key);
     }
-
+    
+    public boolean isAllowSpecificCharacters() {
+        String value = getValue(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS);
+        if (StringUtils.isNotEmpty(value)) {
+            return Boolean.valueOf(value);
+        }       
+        return false; // Default value
+    }
+    
+    public void setAllowSpecificCharacters(boolean isAllow) {
+       setValue(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, isAllow);
+    }
+    
     /**
      * Save the configurations.
      */

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/IJobletProviderService.java

@@ -147,4 +147,6 @@ public interface IJobletProviderService extends IService {
 
     public void updateJobleModifiedRelated(Item item, String oldName, String newName);
 
+    public IProcess getJobletGEFProcessFromNode(INode node);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java

@@ -258,6 +258,8 @@ public interface IRunProcessService extends IService {
 
     public boolean isExcludeDeletedItems(Property property);
 
+    public boolean getMavenPrefOptionStatus(String prefName);
+
     public static IRunProcessService get() {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
             return GlobalServiceRegister.getDefault().getService(IRunProcessService.class);

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/preference/SpecificSettingPreferencePage.java

@@ -12,17 +12,17 @@
 // ============================================================================
 package org.talend.core.ui.preference;
 
-import org.eclipse.jface.preference.BooleanFieldEditor;
 import org.eclipse.jface.preference.FieldEditorPreferencePage;
 import org.eclipse.ui.IWorkbench;
 import org.eclipse.ui.IWorkbenchPreferencePage;
-import org.talend.core.model.repository.IRepositoryPrefConstants;
 import org.talend.core.ui.CoreUIPlugin;
 
 public class SpecificSettingPreferencePage extends FieldEditorPreferencePage implements IWorkbenchPreferencePage {
 
     public SpecificSettingPreferencePage() {
         setPreferenceStore(CoreUIPlugin.getDefault().getPreferenceStore());
+        setDescription("Specific settings"); 
+        noDefaultAndApplyButton();
     }
 
     /*
@@ -32,8 +32,6 @@ public class SpecificSettingPreferencePage extends FieldEditorPreferencePage imp
      */
     @Override
     protected void createFieldEditors() {
-        addField(new BooleanFieldEditor(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS,
-                "Allow specific characters (UTF8,...) for columns of schemas", getFieldEditorParent()));
 
     }
 

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/preference/metadata/MetadataPreferenceInitializer.java

@@ -14,7 +14,6 @@ package org.talend.core.ui.preference.metadata;
 
 import org.eclipse.core.runtime.preferences.AbstractPreferenceInitializer;
 import org.eclipse.jface.preference.IPreferenceStore;
-import org.talend.core.model.repository.IRepositoryPrefConstants;
 import org.talend.core.prefs.ITalendCorePrefConstants;
 import org.talend.core.ui.CoreUIPlugin;
 
@@ -27,8 +26,6 @@ public class MetadataPreferenceInitializer extends AbstractPreferenceInitializer
     public void initializeDefaultPreferences() {
         IPreferenceStore store = CoreUIPlugin.getDefault().getPreferenceStore();
         store.setDefault(ITalendCorePrefConstants.MAXIMUM_AMOUNT_OF_COLUMNS_FOR_XML, 500);
-        store.setDefault(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
-
     }
 
 }

main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java

@@ -25,8 +25,10 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.jar.JarEntry;
 import java.util.jar.JarOutputStream;
@@ -78,8 +80,6 @@ import org.talend.core.language.LanguageManager;
 import org.talend.core.model.components.ComponentCategory;
 import org.talend.core.model.components.EComponentType;
 import org.talend.core.model.components.IComponent;
-import org.talend.core.model.components.IComponentsFactory;
-import org.talend.core.model.components.IComponentsService;
 import org.talend.core.model.context.ContextUtils;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.Project;
@@ -98,10 +98,8 @@ import org.talend.core.model.process.JobInfo;
 import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.process.ReplaceNodesInProcessProvider;
 import org.talend.core.model.properties.Item;
-import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
-import org.talend.core.model.relationship.Relation;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryObject;
@@ -114,6 +112,7 @@ import org.talend.core.model.routines.RoutinesUtil;
 import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.repository.model.ProxyRepositoryFactory;
 import org.talend.core.runtime.CoreRuntimePlugin;
+import org.talend.core.runtime.maven.MavenConstants;
 import org.talend.core.runtime.process.ITalendProcessJavaProject;
 import org.talend.core.runtime.process.LastGenerationInfo;
 import org.talend.core.runtime.process.TalendProcessArgumentConstant;
@@ -192,12 +191,10 @@ public class ProcessorUtilities {
 
     private static final Set<ModuleNeeded> retrievedJarsForCurrentBuild = new HashSet<ModuleNeeded>();
 
-    private static final Set<String> esbJobs = new HashSet<String>();
+    private static final Map<String, Integer> esbJobs = new HashMap<String, Integer>();
 
     private static boolean isDebug = false;
 
-    private static boolean isCIMode = false;
-
     private static boolean isDynamicJobAndCITest = false;
 
     private static JobInfo mainJobInfo;
@@ -441,6 +438,81 @@ public class ProcessorUtilities {
         return false;
     }
 
+    public static boolean checkProcessLoopDependencies(IProcess mainProcess, String id, String version,
+            LinkedList<String> pathlink, Map<String, String> idToLatestVersion) {
+        if (ItemCacheManager.LATEST_VERSION.contains(version)) {
+            if (idToLatestVersion.get(id) == null) {
+                ProcessItem processItem = ItemCacheManager.getProcessItem(id);
+                version = processItem.getProperty().getVersion();
+                idToLatestVersion.put(id, version);
+            } else {
+                version = idToLatestVersion.get(id);
+            }
+        }
+        String pathNode = id + "-" + version;
+        if (pathlink.contains(pathNode)) {
+            return true;
+        }
+        pathlink.add(pathNode);
+
+        boolean hasLoop = false;
+        List<? extends INode> processNodes = mainProcess.getProcessNodes();
+        for (INode node : processNodes) {
+            if (!node.isActivate()) {
+                continue;
+            }
+
+            IElementParameter processIdParam = node.getElementParameter("PROCESS_TYPE_PROCESS");
+            if (processIdParam != null && StringUtils.isNotBlank((String) processIdParam.getValue())) {
+                String jobIds = (String) processIdParam.getValue();
+                String subNodeversion = (String) node.getElementParameter("PROCESS_TYPE_VERSION").getValue();
+                for (String jobId : jobIds.split(ProcessorUtilities.COMMA)) {
+                    if (StringUtils.isBlank(jobId)) {
+                        continue;
+                    }
+                    ProcessItem processItem = ItemCacheManager.getProcessItem(jobId, subNodeversion);
+                    if (processItem != null) {
+                        IDesignerCoreService service = CorePlugin.getDefault().getDesignerCoreService();
+                        IProcess subProcess = service.getProcessFromProcessItem(processItem);
+                        if (subProcess != null) {
+                            hasLoop = checkProcessLoopDependencies(subProcess, jobId, subNodeversion, pathlink,
+                                    idToLatestVersion);
+                        }
+                    }
+                    if (hasLoop) {
+                        break;
+                    }
+                }
+            } else {
+                if (GlobalServiceRegister.getDefault().isServiceRegistered(IJobletProviderService.class)) {
+                    IJobletProviderService jobletService = GlobalServiceRegister.getDefault()
+                            .getService(IJobletProviderService.class);
+                    if (jobletService != null) {
+                        IProcess jobletProcess = jobletService.getJobletGEFProcessFromNode(node);
+                        if (jobletProcess != null) {
+                            String jobletId = jobletProcess.getId();
+                            IElementParameter projectTecNameParam = jobletProcess.getElementParameter("PROJECT_TECHNICAL_NAME");
+                            if (projectTecNameParam != null && StringUtils.isNotBlank((String) projectTecNameParam.getValue())) {
+                                jobletId = projectTecNameParam.getValue() + ":" + jobletId;
+                            }
+                            hasLoop = checkProcessLoopDependencies(jobletProcess, jobletId, jobletProcess.getVersion(), pathlink,
+                                    idToLatestVersion);
+                        }
+                    }
+                }
+            }
+            if (hasLoop) {
+                break;
+            }
+        }
+
+        if (!hasLoop) {
+            pathlink.removeLast();
+        }
+
+        return hasLoop;
+    }
+
     private static IProcessor generateCode(IProcessor processor2, JobInfo jobInfo, String selectedContextName,
             boolean statistics, boolean trace, boolean needContext, int option, IProgressMonitor progressMonitor)
             throws ProcessorException {
@@ -540,11 +612,12 @@ public class ProcessorUtilities {
         jobInfo.setProcessor(processor);
 
         if (isMainJob && selectedProcessItem != null) {
-            Relation mainRelation = new Relation();
-            mainRelation.setId(jobInfo.getJobId());
-            mainRelation.setVersion(jobInfo.getJobVersion());
-            mainRelation.setType(RelationshipItemBuilder.JOB_RELATION);
-            hasLoopDependency = checkLoopDependencies(mainRelation, new HashMap<String, String>());
+            if (!IRunProcessService.get().getMavenPrefOptionStatus(MavenConstants.SKIP_LOOP_DEPENDENCY_CHECK)) {
+                Property property = selectedProcessItem.getProperty();
+                String jobId = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel() + ":" + property.getId();
+                hasLoopDependency = checkProcessLoopDependencies(currentProcess, jobId, property.getVersion(),
+                        new LinkedList<String>(), new HashMap<String, String>());
+            }
             // clean the previous code in case it has deleted subjob
             cleanSourceFolder(progressMonitor, currentProcess, processor);
         }
@@ -647,233 +720,6 @@ public class ProcessorUtilities {
         return processor;
     }
 
-    public static boolean checkLoopDependencies(Relation mainJobInfo, Map<String, String> idToLastestVersionMap)
-            throws ProcessorException {
-        List<Relation> itemsJobRelatedTo = getItemsRelation(mainJobInfo, idToLastestVersionMap);
-        List<Relation> relationChecked = new ArrayList<>();
-        relationChecked.add(mainJobInfo);
-        return checkLoopDependencies(mainJobInfo, mainJobInfo, itemsJobRelatedTo, relationChecked, idToLastestVersionMap);
-    }
-
-    private static boolean checkLoopDependencies(Relation mainRelation, Relation currentRelation,
-            List<Relation> itemsJobRelatedTo,
-            List<Relation> relationChecked, Map<String, String> idToLastestVersionMap) throws ProcessorException {
-        boolean hasDependency = false;
-        for (Relation relation : itemsJobRelatedTo) {
-            try {
-                // means the tRunjob deactivate, or one of the specific version tRunjon deactivate, skip
-                Map<String, Set<String>> actTrunjobHM = getActivateTRunjobMap(currentRelation.getId(),
-                        currentRelation.getVersion());
-                if (actTrunjobHM.get(relation.getId()) == null
-                        || !actTrunjobHM.get(relation.getId()).contains(relation.getVersion())) {
-                    continue;
-                }
-            } catch (Exception e) {
-                throw new ProcessorException(e);
-            }
-
-            hasDependency = relation.getId().equals(mainRelation.getId())
-                    && relation.getVersion().equals(mainRelation.getVersion());
-            if (!hasDependency) {
-                List<Relation> itemsChildJob = getItemsRelation(relation, idToLastestVersionMap);
-                if (!relationChecked.contains(relation)) {
-                    relationChecked.add(relation);
-                    hasDependency = checkLoopDependencies(mainRelation, relation, itemsChildJob, relationChecked,
-                            idToLastestVersionMap);
-                }
-                if (!hasDependency) {
-                    for (Relation childRelation : itemsChildJob) {
-
-                        try {
-                            // means the tRunjob deactivate, or one of the specific version tRunjon deactivate, skip
-                            Map<String, Set<String>> activateTRunjobMap = getActivateTRunjobMap(relation.getId(),
-                                    relation.getVersion());
-                            if (activateTRunjobMap.get(childRelation.getId()) == null
-                                    || !activateTRunjobMap.get(childRelation.getId()).contains(childRelation.getVersion())) {
-                                continue;
-                            }
-                        } catch (Exception e) {
-                            throw new ProcessorException(e);
-                        }
-
-                        hasDependency = checkLoopDependencies(childRelation, idToLastestVersionMap);
-                        if (hasDependency) {
-                            break;
-                        }
-                    }
-                }
-            }
-            if (hasDependency) {
-                break;
-            }
-        }
-
-        return hasDependency;
-    }
-
-    private static Map<String, Set<String>> getActivateTRunjobMap(String id, String version) throws PersistenceException {
-        Map<String, Set<String>> actTrunjobHM = new HashMap<String, Set<String>>();
-        ProcessType processType = null;
-        try {
-            IRepositoryViewObject currentJobObject = ProxyRepositoryFactory.getInstance().getSpecificVersion(id, version, true);
-            if (currentJobObject != null) {
-                Item item = currentJobObject.getProperty().getItem();
-                if (item instanceof ProcessItem) {
-                    processType = ((ProcessItem) item).getProcess();
-                } else if (item instanceof JobletProcessItem) {
-                    processType = ((JobletProcessItem) item).getJobletProcess();
-                }
-            }
-        } catch (PersistenceException e) {
-            ExceptionHandler.process(e);
-        }
-        if (processType != null) {
-            List<Project> allProjects = new ArrayList<Project>();
-            allProjects.add(ProjectManager.getInstance().getCurrentProject());
-            allProjects.addAll(ProjectManager.getInstance().getAllReferencedProjects());
-
-            List<String> jobletsComponentsList = new ArrayList<String>();
-            IComponentsFactory componentsFactory = null;
-            if (GlobalServiceRegister.getDefault().isServiceRegistered(IComponentsService.class)) {
-                IComponentsService compService = GlobalServiceRegister.getDefault()
-                        .getService(IComponentsService.class);
-                if (compService != null) {
-                    componentsFactory = compService.getComponentsFactory();
-                    for (IComponent component : componentsFactory.readComponents()) {
-                        if (component.getComponentType() == EComponentType.JOBLET) {
-                            jobletsComponentsList.add(component.getName());
-                        }
-                    }
-                }
-            }
-
-            String jobletPaletteType = null;
-            String frameWork = processType.getFramework();
-            if (StringUtils.isBlank(frameWork)) {
-                jobletPaletteType = ComponentCategory.CATEGORY_4_DI.getName();
-            } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK)) {
-                jobletPaletteType = ComponentCategory.CATEGORY_4_SPARK.getName();
-            } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK_STREAMING)) {
-                jobletPaletteType = ComponentCategory.CATEGORY_4_SPARKSTREAMING.getName();
-            }
-
-            for (Object nodeObject : processType.getNode()) {
-                NodeType node = (NodeType) nodeObject;
-                // not tRunjob && not joblet then continue
-                if (!node.getComponentName().equals("tRunJob") && !jobletsComponentsList.contains(node.getComponentName())) { // $NON-NLS-1$
-                    continue;
-                }
-                boolean nodeActivate = true;
-                String processIds = null;
-                String processVersion = null;
-                for (Object elementParam : node.getElementParameter()) {
-                    ElementParameterType elemParamType = (ElementParameterType) elementParam;
-                    if ("PROCESS:PROCESS_TYPE_PROCESS".equals(elemParamType.getName())) { // $NON-NLS-1$
-                        processIds = elemParamType.getValue();
-                        if (StringUtils.isNotBlank(processIds)) {
-                            for (String jobId : processIds.split(ProcessorUtilities.COMMA)) {
-                                if (actTrunjobHM.get(jobId) == null) {
-                                    actTrunjobHM.put(jobId, new HashSet<String>());
-                                }
-                            }
-                        }
-                    } else if ("PROCESS:PROCESS_TYPE_VERSION".equals(elemParamType.getName()) // $NON-NLS-1$
-                            || "PROCESS_TYPE_VERSION".equals(elemParamType.getName())) { // $NON-NLS-1$
-                        processVersion = elemParamType.getValue();
-                    } else if ("ACTIVATE".equals(elemParamType.getName())) { // $NON-NLS-1$
-                        nodeActivate = Boolean.parseBoolean(elemParamType.getValue());
-                    }
-                }
-
-                if (nodeActivate) {
-                    if (StringUtils.isNotBlank(processIds)) {
-                        for (String jobId : processIds.split(ProcessorUtilities.COMMA)) {
-                            String actualVersion = processVersion;
-                            if (RelationshipItemBuilder.LATEST_VERSION.equals(processVersion)) {
-                                for (Project project : allProjects) {
-                                    IRepositoryViewObject lastVersion = null;
-                                    lastVersion = ProxyRepositoryFactory.getInstance().getLastVersion(project, jobId);
-                                    if (lastVersion != null) {
-                                        actualVersion = lastVersion.getVersion();
-                                        break;
-                                    }
-                                }
-                            }
-                            if (actTrunjobHM.get(jobId) != null) {
-                                actTrunjobHM.get(jobId).add(actualVersion);
-                            }
-
-                        }
-                    } else if (componentsFactory != null && jobletPaletteType != null) {
-                        // for joblet
-                        IComponent cc = componentsFactory.get(node.getComponentName(), jobletPaletteType);
-                        if (GlobalServiceRegister.getDefault().isServiceRegistered(IJobletProviderService.class)) {
-                            IJobletProviderService jobletService = GlobalServiceRegister.getDefault()
-                                    .getService(IJobletProviderService.class);
-                            Property property = jobletService.getJobletComponentItem(cc);
-                            if (property != null && StringUtils.isNotBlank(property.getId())) {
-                                String jobletId = property.getId();
-                                if (actTrunjobHM.get(jobletId) == null) {
-                                    actTrunjobHM.put(jobletId, new HashSet<String>());
-                                }
-                                String actualVersion = processVersion;
-                                if (RelationshipItemBuilder.LATEST_VERSION.equals(processVersion)) {
-                                    for (Project project : allProjects) {
-                                        IRepositoryViewObject lastVersion = null;
-                                        lastVersion = ProxyRepositoryFactory.getInstance().getLastVersion(project, jobletId);
-                                        if (lastVersion != null) {
-                                            actualVersion = lastVersion.getVersion();
-                                            break;
-                                        }
-                                    }
-                                }
-
-                                actTrunjobHM.get(jobletId).add(actualVersion);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        return actTrunjobHM;
-    }
-
-    private static List<Relation> getItemsRelation(Relation mainJobInfo, Map<String, String> idToLastestVersionMap) throws ProcessorException {
-        List<Relation> itemsJobRelatedTo = new ArrayList<Relation>();
-        try {
-            List<Project> allProjects = new ArrayList<Project>();
-            allProjects.add(ProjectManager.getInstance().getCurrentProject());
-            allProjects.addAll(ProjectManager.getInstance().getAllReferencedProjects());
-            RelationshipItemBuilder instance = RelationshipItemBuilder.getInstance();
-            if (instance != null) {
-                itemsJobRelatedTo.addAll(instance.getItemsChildRelatedTo(mainJobInfo.getId(), mainJobInfo.getVersion(),
-                        mainJobInfo.getType(), RelationshipItemBuilder.JOB_RELATION));
-                itemsJobRelatedTo.addAll(instance.getItemsChildRelatedTo(mainJobInfo.getId(), mainJobInfo.getVersion(),
-                        mainJobInfo.getType(), RelationshipItemBuilder.JOBLET_RELATION));
-                for (Relation relation : itemsJobRelatedTo) {
-                    if (relation.getVersion().equals(RelationshipItemBuilder.LATEST_VERSION)) {
-                        if (idToLastestVersionMap.containsKey(relation.getId())) {
-                            relation.setVersion(idToLastestVersionMap.get(relation.getId()));
-                        } else {
-                            for (Project project : allProjects) {
-                                IRepositoryViewObject lastVersion =
-                                        ProxyRepositoryFactory.getInstance().getLastVersion(project, relation.getId());
-                                if (lastVersion != null) {
-                                    relation.setVersion(lastVersion.getVersion());
-                                    idToLastestVersionMap.put(relation.getId(), relation.getVersion());
-                                    break;
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        } catch (PersistenceException e) {
-            throw new ProcessorException(e);
-        }
-
-        return itemsJobRelatedTo;
-    }
 
     private static void setNeededResources(final Map<String, Object> argumentsMap, JobInfo jobInfo) {
         argumentsMap.put(TalendProcessArgumentConstant.ARG_NEED_XMLMAPPINGS,
@@ -1267,11 +1113,11 @@ public class ProcessorUtilities {
             }
 
             if (isMainJob && selectedProcessItem != null) {
-                Relation mainRelation = new Relation();
-                mainRelation.setId(jobInfo.getJobId());
-                mainRelation.setVersion(jobInfo.getJobVersion());
-                mainRelation.setType(RelationshipItemBuilder.JOB_RELATION);
-                hasLoopDependency = checkLoopDependencies(mainRelation, new HashMap<String, String>());
+                if (!IRunProcessService.get().getMavenPrefOptionStatus(MavenConstants.SKIP_LOOP_DEPENDENCY_CHECK)) {
+                    Property property = selectedProcessItem.getProperty();
+                    hasLoopDependency = checkProcessLoopDependencies(currentProcess, property.getId(), property.getVersion(),
+                            new LinkedList<String>(), new HashMap<String, String>());
+                }
                 // clean the previous code in case it has deleted subjob
                 cleanSourceFolder(progressMonitor, currentProcess, processor);
             }
@@ -1317,7 +1163,7 @@ public class ProcessorUtilities {
             checkMetadataDynamic(currentProcess, jobInfo);
 
             int options = TalendProcessOptionConstants.MODULES_DEFAULT;
-            if (isCIMode && BitwiseOptionUtils.containOption(option, GENERATE_MAIN_ONLY)) {
+            if (isCIMode() && BitwiseOptionUtils.containOption(option, GENERATE_MAIN_ONLY)) {
                 options |= TalendProcessOptionConstants.MODULES_WITH_CHILDREN;
             }
             Set<ModuleNeeded> neededLibraries = new HashSet<>();
@@ -1802,13 +1648,16 @@ public class ProcessorUtilities {
                         }
                     }
                 }
-                if (isEsbComponentName(componentName)) {
-                    addEsbJob(jobInfo);
-                }
             }
         }
     }
 
+    private static void recordESBIncludingFlag(JobInfo jobInfo, int esbIncludingOption) {
+        int includeESBFlag = jobInfo.getIncludeESBFlag();
+        includeESBFlag |= esbIncludingOption;
+        jobInfo.setIncludeESBFlag(includeESBFlag);
+    }
+
     static void setGenerationInfoWithChildrenJob(INode node, JobInfo jobInfo, final JobInfo subJobInfo) {
         final LastGenerationInfo generationInfo = LastGenerationInfo.getInstance();
 
@@ -1917,6 +1766,13 @@ public class ProcessorUtilities {
 
     static boolean hasLoopDependency = false;
 
+    private static void resetBuildFlagsAndCaches() {
+        jobList.clear();
+        esbJobs.clear();
+        hasLoopDependency = false;
+        mainJobInfo = null;
+    }
+
     /**
      * This function will generate the code of the process and all of this sub process.
      *
@@ -1931,15 +1787,10 @@ public class ProcessorUtilities {
         if (monitors == null) {
             monitor = new NullProgressMonitor();
         }
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         JobInfo jobInfo = new JobInfo(processName, contextName, version);
         IProcessor process = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return process;
     }
 
@@ -1959,14 +1810,9 @@ public class ProcessorUtilities {
         }
         JobInfo jobInfo = new JobInfo(processId, contextName, version);
         jobInfo.setApplyContextToChildren(applyContextToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor process = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return process;
     }
 
@@ -1981,14 +1827,9 @@ public class ProcessorUtilities {
         }
         JobInfo jobInfo = new JobInfo(process, contextName);
         jobInfo.setApplyContextToChildren(applyContextToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor result = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return result;
     }
 
@@ -2003,15 +1844,10 @@ public class ProcessorUtilities {
         }
         JobInfo jobInfo = new JobInfo(process, contextName, version);
         jobInfo.setApplyContextToChildren(applyContextToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor result =
                 generateCode(jobInfo, contextName, statistics, trace, needContext, GENERATE_ALL_CHILDS, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return result;
     }
 
@@ -2035,14 +1871,9 @@ public class ProcessorUtilities {
                 ProcessUtils.isOptionChecked(argumentsMap, TalendProcessArgumentConstant.ARG_NEED_CONTEXT);
         int option = ProcessUtils.getOptionValue(argumentsMap, TalendProcessArgumentConstant.ARG_GENERATE_OPTION, 0);
 
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor result = generateCode(jobInfo, contextName, statistics, trace, needContext, option, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return result;
     }
 
@@ -2060,28 +1891,18 @@ public class ProcessorUtilities {
             JobInfo jobInfo = new JobInfo(process, contextName, version);
             jobInfo.setContext(context);
             jobInfo.setApplyContextToChildren(applyContextToChildren);
-            jobList.clear();
-            esbJobs.clear();
-            hasLoopDependency = false;
-            mainJobInfo = null;
+            resetBuildFlagsAndCaches();
             result = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
-            jobList.clear();
-            hasLoopDependency = false;
-            mainJobInfo = null;
+            resetBuildFlagsAndCaches();
         }
         return result;
     }
 
     public static IProcessor generateCode(ProcessItem process, String contextName, boolean statistics, boolean trace)
             throws ProcessorException {
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor returnValue = generateCode(process, contextName, statistics, trace, false);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return returnValue;
     }
 
@@ -2109,29 +1930,19 @@ public class ProcessorUtilities {
             jobInfo = new JobInfo(process, context);
         }
         jobInfo.setApplyContextToChildren(applyToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor genCode = generateCode(jobInfo, context.getName(), statistics, trace, contextProperties,
                 GENERATE_ALL_CHILDS, new NullProgressMonitor());
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return genCode;
     }
 
     public static IProcessor generateCode(IProcess process, IContext context, boolean statistics, boolean trace,
             boolean properties) throws ProcessorException {
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor returnValue =
                 generateCode(process, context, statistics, trace, properties, new NullProgressMonitor());
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return returnValue;
     }
 
@@ -2159,15 +1970,10 @@ public class ProcessorUtilities {
         } else {
             jobInfo = new JobInfo(process, context);
         }
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor genCode = generateCode(jobInfo, context.getName(), statistics, trace, properties,
                 GENERATE_ALL_CHILDS, progressMonitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return genCode;
     }
 
@@ -2210,15 +2016,10 @@ public class ProcessorUtilities {
                 + (jobInfo.getJobName() != null ? jobInfo.getJobName() : jobInfo.getJobId());
         TimeMeasure.begin(timeMeasureGenerateCodesId);
 
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor genCode = generateCode(processor, jobInfo, context.getName(), statistics, trace, properties,
                 GENERATE_ALL_CHILDS, progressMonitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
 
         TimeMeasure.end(timeMeasureGenerateCodesId);
         // if active before, not disable and active still.
@@ -2241,15 +2042,10 @@ public class ProcessorUtilities {
         }
         // achen modify to fix 0006107
         JobInfo jobInfo = new JobInfo(process, context);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor genCode = generateCode(jobInfo, context.getName(), statistics, trace, properties, option,
                 new NullProgressMonitor());
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return genCode;
     }
 
@@ -2264,14 +2060,9 @@ public class ProcessorUtilities {
         }
         JobInfo jobInfo = new JobInfo(process, contextName);
         jobInfo.setApplyContextToChildren(applyContextToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor result = generateCode(jobInfo, contextName, statistics, trace, true, option, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return result;
     }
 
@@ -2613,6 +2404,8 @@ public class ProcessorUtilities {
         } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK_STREAMING)) {
             jobletPaletteType = ComponentCategory.CATEGORY_4_SPARKSTREAMING.getName();
         }
+
+        boolean hasChildrenIncludeESB = false;
         for (NodeType node : nodes) {
             boolean activate = true;
             // check if node is active at least.
@@ -2627,8 +2420,8 @@ public class ProcessorUtilities {
                 continue;
             }
 
-            if (isEsbComponentName(node.getComponentName())) {
-                addEsbJob(parentJobInfo);
+            if (!firstChildOnly && isEsbComponentName(node.getComponentName())) {
+                recordESBIncludingFlag(parentJobInfo, TalendProcessOptionConstants.ISESB_CURRENT_INCLUDE);
             }
 
             boolean isCTalendJob = "cTalendJob".equalsIgnoreCase(node.getComponentName());
@@ -2657,6 +2450,18 @@ public class ProcessorUtilities {
                                 jobInfo.setFatherJobInfo(parentJobInfo);
                                 if (!firstChildOnly) {
                                     getAllJobInfo(processItem.getProcess(), jobInfo, jobInfos, firstChildOnly, includeJoblet);
+                                    if (jobInfo.getIncludeESBFlag() >= 2) {
+                                        hasChildrenIncludeESB = true;
+                                    }
+                                }
+                            } else {
+                                Optional<JobInfo> infoOptional = jobInfos.stream().filter(info -> info.equals(jobInfo))
+                                        .findFirst();
+                                if (infoOptional.isPresent()) {
+                                    JobInfo matchJobInfo = infoOptional.get();
+                                    if (matchJobInfo.getIncludeESBFlag() >= 2) {
+                                        hasChildrenIncludeESB = true;
+                                    }
                                 }
                             }
                         }
@@ -2708,6 +2513,16 @@ public class ProcessorUtilities {
                 }
             }
         }
+
+        // checked done set the esb including option
+        if (!firstChildOnly) {
+            recordESBIncludingFlag(parentJobInfo, TalendProcessOptionConstants.ISESB_CHECKED);
+            if (hasChildrenIncludeESB) {
+                recordESBIncludingFlag(parentJobInfo, TalendProcessOptionConstants.ISESB_CHILDREN_INCLUDE);
+            }
+            esbJobs.put(esbJobKey(parentJobInfo.getJobId(), parentJobInfo.getJobVersion()), parentJobInfo.getIncludeESBFlag());
+        }
+
         return jobInfos;
     }
 
@@ -2924,9 +2739,21 @@ public class ProcessorUtilities {
     }
 
     public static boolean isEsbJob(IProcess process, boolean checkCurrentProcess) {
+        // get includeESBFlag from cache
+        if (process instanceof IProcess2) {
+            Property property = ((IProcess2) process).getProperty();
+            String esbJobKey = esbJobKey(property.getId(), property.getVersion());
+            if (esbJobs.get(esbJobKey) != null) {
+                Integer esbOptions = esbJobs.get(esbJobKey);
+                if (BitwiseOptionUtils.containOption(esbOptions, TalendProcessOptionConstants.ISESB_CHECKED)) {
+                    return checkCurrentProcess
+                            ? BitwiseOptionUtils.containOption(esbOptions, TalendProcessOptionConstants.ISESB_CURRENT_INCLUDE)
+                            : esbOptions > 2;
+                }
+            }
+        }
 
         if (process instanceof IProcess2) {
-
             if (checkCurrentProcess) {
                 for (INode n : process.getGraphicalNodes()) {
                     if (isEsbComponentName(n.getComponent().getName())) {
@@ -2952,17 +2779,6 @@ public class ProcessorUtilities {
         return false;
     }
 
-    private static void addEsbJob(JobInfo jobInfo) {
-        if (esbJobs.contains(esbJobKey(jobInfo.getJobId(), jobInfo.getJobVersion()))) {
-            return;
-        }
-        
-        esbJobs.add(esbJobKey(jobInfo.getJobId(), jobInfo.getJobVersion()));
-        if (jobInfo.getFatherJobInfo() != null) {
-            addEsbJob(jobInfo.getFatherJobInfo());
-        }
-    }
-
     private static String esbJobKey(String processId, String version) {
         return processId + "_" + version;
     }
@@ -3044,13 +2860,11 @@ public class ProcessorUtilities {
     }
 
     public static boolean isCIMode() {
-        return isCIMode;
+        // if it's CI mode , then the system property of maven.local.repository will store the value of studio
+        // m2 path,otherwise it's null
+        return System.getProperty("maven.local.repository") != null;
     }
 
-    public static void setCIMode(boolean isCIMode) {
-        ProcessorUtilities.isCIMode = isCIMode;
-    }
-    
     public static boolean hasRoutelet(ProcessItem prItem, String routelet) {
         EList<NodeType> nodeList = prItem.getProcess().getNode();
 

main/plugins/org.talend.designer.maven.aether/.classpath

@@ -26,7 +26,6 @@
 	<classpathentry exported="true" kind="lib" path="lib/maven-model-builder-3.2.1.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/maven-repository-metadata-3.2.1.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/plexus-interpolation-1.19.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/plexus-utils-3.0.17.jar"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
 	<classpathentry kind="src" path="src/main/java"/>

main/plugins/org.talend.designer.maven.aether/META-INF/MANIFEST.MF

@@ -16,7 +16,6 @@ Bundle-ClassPath: .,
  lib/maven-model-builder-3.2.1.jar,
  lib/maven-repository-metadata-3.2.1.jar,
  lib/plexus-interpolation-1.19.jar,
- lib/plexus-utils-3.0.17.jar,
  lib/commons-codec.jar,
  lib/httpclient.jar,
  lib/httpcore.jar,

main/plugins/org.talend.designer.maven.tos/pom.xml

@@ -40,6 +40,12 @@
             <version>${project.version}</version>
             <type>pom</type>
         </dependency>
+        <dependency>
+            <groupId>org.talend.studio</groupId>
+            <artifactId>studio-log4j-dependencies-tos</artifactId>
+            <version>${project.version}</version>
+            <type>pom</type>
+        </dependency>
         <dependency>
 			<groupId>org.talend.studio</groupId>
 			<artifactId>studio-maven-repository-build</artifactId>

main/plugins/org.talend.designer.maven.tos/resources/build/plugins/build-helper-maven-plugin-3-0-0/pom.xml

@@ -14,6 +14,17 @@
 			<groupId>org.codehaus.mojo</groupId>
 			<artifactId>build-helper-maven-plugin</artifactId>
 			<version>3.0.0</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.apache.maven.shared</groupId>
+					<artifactId>maven-shared-utils</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.maven.shared</groupId>
+			<artifactId>maven-shared-utils</artifactId>
+			<version>3.3.3</version>
 		</dependency>
 	</dependencies>
 	<build>

main/plugins/org.talend.designer.maven.tos/resources/build/plugins/karaf-maven-plugin-4-2-7/pom.xml

@@ -19,6 +19,10 @@
 					<groupId>org.apache.httpcomponents</groupId>
 					<artifactId>httpclient</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>org.apache.maven.shared</groupId>
+					<artifactId>maven-shared-utils</artifactId>
+				</exclusion>
 			</exclusions>
 		</dependency>
 		<dependency>
@@ -46,6 +50,17 @@
 			<groupId>org.apache.httpcomponents</groupId>
 			<artifactId>httpclient</artifactId>
 			<version>4.5.13</version>
+			<exclusions>
+				<exclusion>
+					<groupId>commons-codec</groupId>
+					<artifactId>commons-codec</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>commons-codec</groupId>
+			<artifactId>commons-codec</artifactId>
+			<version>1.15</version>
 		</dependency>
 	</dependencies>
 	<build>

main/plugins/org.talend.designer.maven.tos/resources/build/pom.xml

@@ -23,21 +23,45 @@
 			<groupId>org.apache.maven.plugins</groupId>
 			<artifactId>maven-clean-plugin</artifactId>
 			<version>3.0.0</version>
+			<exclusions>
+    			<exclusion>
+     				<groupId>org.apache.maven.shared</groupId>
+     				<artifactId>maven-shared-utils</artifactId>
+    			</exclusion>
+   			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.maven.plugins</groupId>
 			<artifactId>maven-compiler-plugin</artifactId>
 			<version>3.6.1</version>
+			<exclusions>
+    			<exclusion>
+     				<groupId>org.apache.maven.shared</groupId>
+     				<artifactId>maven-shared-utils</artifactId>
+    			</exclusion>
+   			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.maven.plugins</groupId>
 			<artifactId>maven-resources-plugin</artifactId>
 			<version>3.0.2</version>
+			<exclusions>
+    			<exclusion>
+     				<groupId>org.apache.maven.shared</groupId>
+     				<artifactId>maven-shared-utils</artifactId>
+    			</exclusion>
+   			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.maven.plugins</groupId>
 			<artifactId>maven-jar-plugin</artifactId>
 			<version>3.2.0</version>
+			<exclusions>
+    			<exclusion>
+     				<groupId>org.apache.maven.shared</groupId>
+     				<artifactId>maven-shared-utils</artifactId>
+    			</exclusion>
+   			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.maven.plugins</groupId>
@@ -48,6 +72,10 @@
 					<groupId>commons-codec</groupId>
 					<artifactId>commons-codec</artifactId>
 				</exclusion>
+				<exclusion>
+     				<groupId>org.apache.maven.shared</groupId>
+     				<artifactId>maven-shared-utils</artifactId>
+    			</exclusion>
 			</exclusions>
 		</dependency>
 		<dependency>

main/plugins/org.talend.designer.maven.tos/resources/log4j/pom.xml

@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>org.talend.studio</groupId>
+		<artifactId>studio-maven-repository-tos</artifactId>
+		<version>7.3.1-PATCH</version>
+		<relativePath>../</relativePath>
+	</parent>
+	<artifactId>studio-log4j-dependencies-tos</artifactId>
+	<packaging>pom</packaging>
+	<dependencies>
+        <!--add log4j dependency https://jira.talendforge.org/browse/TUP-31403  -->
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j-impl</artifactId>
+            <version>2.12.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <version>2.12.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
+            <version>2.12.1</version>
+        </dependency>
+        <dependency>
+	      <groupId>org.apache.logging.log4j</groupId>
+	      <artifactId>log4j-1.2-api</artifactId>
+	      <version>2.12.1</version>
+	    </dependency>
+	</dependencies>
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-dependency-plugin</artifactId>
+				<version>2.10</version>
+				<executions>
+					<execution>
+						<id>copy-dependencies</id>
+						<phase>package</phase>
+						<goals>
+							<goal>copy-dependencies</goal>
+						</goals>
+						<configuration>
+							<includeScope>runtime</includeScope>
+							<outputDirectory>${basedir}/../tmp/repository</outputDirectory>
+							<useRepositoryLayout>true</useRepositoryLayout>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+</project>

main/plugins/org.talend.designer.maven.tos/resources/pom.xml

@@ -16,6 +16,7 @@
         <module>tacokit/pom.xml</module>
         <module>tcompv1/pom.xml</module>
         <module>surefire/pom.xml</module>
+        <module>log4j/pom.xml</module>
         <module>zip/pom.xml</module>
     </modules>
     <properties>

main/plugins/org.talend.designer.maven.tos/resources/tacokit/pom.xml

@@ -10,7 +10,7 @@
 	<artifactId>studio-tacokit-dependencies</artifactId>
 	<packaging>pom</packaging>
 	<properties>
-		<tacokit.components.version>1.20.0</tacokit.components.version>
+		<tacokit.components.version>1.21.0</tacokit.components.version>
 	</properties>
 	<repositories>
 		<repository>
@@ -35,7 +35,18 @@
 		</repository>
 	</repositories>
 
-
+	<dependencyManagement>
+		<dependencies>
+			<dependency>
+				<groupId>org.talend.components</groupId>
+				<artifactId>connectors-se</artifactId>
+				<version>${tacokit.components.version}</version>
+				<type>pom</type>
+				<scope>import</scope>
+			</dependency>
+		</dependencies>
+	</dependencyManagement>
+	
 	<dependencies>
 		<dependency>
 			<groupId>org.talend.components</groupId>

main/plugins/org.talend.designer.maven.tos/resources/tcompv1/pom.xml

@@ -11,7 +11,7 @@
     <packaging>pom</packaging>
     
     <properties>
-        <tcomp.version>1.31.2</tcomp.version>
+        <tcomp.version>1.32.0</tcomp.version>
         <slf4j.version>1.7.25</slf4j.version>
     </properties>
     

main/plugins/org.talend.designer.maven.tos/resources/unzip/pom.xml

@@ -33,7 +33,7 @@
                                 <artifactItem>
                                     <groupId>org.talend.studio</groupId>
                                     <artifactId>studio-maven-repository</artifactId>
-                                    <version>7.2.1.v20190226</version>
+                                    <version>7.3.1.v20210520</version>
                                     <type>zip</type>
                                     <overWrite>true</overWrite>
                                     <outputDirectory>${project.basedir}/../repository</outputDirectory>

main/plugins/org.talend.designer.maven.ui/src/main/java/messages.properties

@@ -15,4 +15,5 @@ MavenProjectSettingPage.filterExampleMessage=Filter examples:\nlabel=myJob
 MavenProjectSettingPage.refModuleText=Set reference project modules in profile
 MavenProjectSettingPage.excludeDeletedItems=Exclude deleted items
 MavenProjectSettingPage.syncAllPomsWarning=Click the Force full re-synchronize poms button to apply the new settings.
-MavenProjectSettingPage.skipFolders=Skip folders
+MavenProjectSettingPage.skipFolders=Skip folders
+BuildProjectSettingPage.allowRecursiveJobs=Allow recursive jobs (Not recommended)

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/setting/project/page/BuildProjectSettingPage.java

@@ -12,15 +12,50 @@
 // ============================================================================
 package org.talend.designer.maven.ui.setting.project.page;
 
+import org.eclipse.jface.preference.IPreferenceStore;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.layout.GridLayout;
+import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Composite;
+import org.talend.core.runtime.maven.MavenConstants;
 import org.talend.core.runtime.projectsetting.EmptyProjectSettingPage;
+import org.talend.designer.maven.DesignerMavenPlugin;
+import org.talend.designer.maven.ui.i18n.Messages;
 
 /**
  * DOC ggu class global comment. Detailled comment
  */
 public class BuildProjectSettingPage extends EmptyProjectSettingPage {
 
+    private IPreferenceStore preferenceStore;
+
+    private Button allowRecursiveJobsCheckbox;
+
     public BuildProjectSettingPage() {
         super();
     }
 
+    @Override
+    protected String getPreferenceName() {
+        return DesignerMavenPlugin.PLUGIN_ID;
+    }
+
+    @Override
+    protected void createFieldEditors() {
+        Composite parent = getFieldEditorParent();
+        parent.setLayout(new GridLayout());
+        preferenceStore = getPreferenceStore();
+        allowRecursiveJobsCheckbox = new Button(parent, SWT.CHECK);
+        allowRecursiveJobsCheckbox.setText(Messages.getString("BuildProjectSettingPage.allowRecursiveJobs")); //$NON-NLS-1$
+        allowRecursiveJobsCheckbox.setSelection(!preferenceStore.getBoolean(MavenConstants.SKIP_LOOP_DEPENDENCY_CHECK));
+    }
+
+    @Override
+    public boolean performOk() {
+        boolean performOk = super.performOk();
+        if (preferenceStore != null) {
+            preferenceStore.setValue(MavenConstants.SKIP_LOOP_DEPENDENCY_CHECK, !allowRecursiveJobsCheckbox.getSelection());
+        }
+        return performOk;
+    }
 }

main/plugins/org.talend.designer.maven/resources/templates/general/pom_codes_template.xml

@@ -17,8 +17,8 @@
 
 	<properties>
 		<talend.codes.package>@CodesPackage@</talend.codes.package>
-        <allow.codes.compile.error>true</allow.codes.compile.error>
-        <maven.compiler.failOnError>${allow.codes.compile.error}</maven.compiler.failOnError>
+        <codes.compile.failOnError>true</codes.compile.failOnError>
+        <maven.compiler.failOnError>${codes.compile.failOnError}</maven.compiler.failOnError>
 	</properties>
 
 	<dependencies>

main/plugins/org.talend.designer.maven/resources/templates/general/pom_project_template.xml

@@ -31,6 +31,11 @@
 					</configuration>
 					<dependencies>
 						<!-- This dependency provides the implementation of compiler "jdt": -->
+						<dependency>
+					       <groupId>org.apache.maven.shared</groupId>
+					       <artifactId>maven-shared-utils</artifactId>
+					       <version>3.3.3</version>
+					    </dependency>
 						<dependency>
 							<groupId>org.eclipse.tycho</groupId>
 							<artifactId>tycho-compiler-jdt</artifactId>
@@ -63,6 +68,11 @@
 							<artifactId>plexus-utils</artifactId>
 							<version>3.0.24</version>
 						</dependency>
+						<dependency>
+					       <groupId>org.apache.maven.shared</groupId>
+					       <artifactId>maven-shared-utils</artifactId>
+					       <version>3.3.3</version>
+					    </dependency>
 					</dependencies>
 				</plugin>
 				<plugin>
@@ -80,6 +90,11 @@
 							<artifactId>commons-compress</artifactId>
 							<version>1.19</version>
 						</dependency>
+						<dependency>
+					       <groupId>org.apache.maven.shared</groupId>
+					       <artifactId>maven-shared-utils</artifactId>
+					       <version>3.3.3</version>
+					    </dependency>
 					</dependencies>
 					<configuration>
 						<archive>
@@ -92,6 +107,11 @@
 					<artifactId>maven-assembly-plugin</artifactId>
 					<version>3.0.0</version>
 					<dependencies>
+						<dependency>
+					       <groupId>org.apache.maven.shared</groupId>
+					       <artifactId>maven-shared-utils</artifactId>
+					       <version>3.3.3</version>
+					    </dependency>
 						<dependency>
 							<groupId>org.codehaus.plexus</groupId>
 							<artifactId>plexus-utils</artifactId>
@@ -107,7 +127,7 @@
 				<plugin>
 					<groupId>org.apache.maven.plugins</groupId>
 					<artifactId>maven-surefire-plugin</artifactId>
-					<version>2.20</version>
+					<version>2.12.4</version>
 					<dependencies>
 						<dependency>
 							<groupId>org.codehaus.plexus</groupId>

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/launch/MavenCommandLauncher.java

@@ -97,7 +97,7 @@ public abstract class MavenCommandLauncher {
 
     private Map<String, Object> argumentsMap;
 
-    private static final String REGEX_TEST_CASE_FAILURES_STR = "\\[ERROR\\] Tests run:.*\\[ERROR\\] There are test\\p{Print}+\n";
+    private static final String REGEX_TEST_CASE_FAILURES_STR = "\\[ERROR\\] There are test failures.*\n";
 
     private static final Pattern REGEX_TEST_CASE_FAILURES = Pattern.compile(REGEX_TEST_CASE_FAILURES_STR, Pattern.DOTALL);
 

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/AbstractMavenCodesTemplatePom.java

@@ -30,6 +30,7 @@ import org.talend.core.model.properties.Property;
 import org.talend.core.runtime.projectsetting.IProjectSettingTemplateConstants;
 import org.talend.designer.maven.template.MavenTemplateManager;
 import org.talend.designer.maven.utils.PomUtil;
+import org.talend.designer.runprocess.IRunProcessService;
 
 /**
  * DOC ggu class global comment. Detailled comment
@@ -116,7 +117,13 @@ public abstract class AbstractMavenCodesTemplatePom extends AbstractMavenGeneral
                 } else {
                     isDeployed = true;
                 }
-                if (ignoreModuleInstallationStatus() || isDeployed) {
+                boolean isCIMode = false;
+                if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
+                    IRunProcessService runProcessService = GlobalServiceRegister.getDefault()
+                            .getService(IRunProcessService.class);
+                    isCIMode = runProcessService.isCIMode();
+                }
+                if (isCIMode || ignoreModuleInstallationStatus() || isDeployed) {
                     dependency = PomUtil.createModuleDependency(module.getMavenUri());
                     if (module.isExcluded())
                         dependency.setScope("provided");

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/PomIdsHelper.java

@@ -315,6 +315,12 @@ public class PomIdsHelper {
         return manager.getBoolean(MavenConstants.EXCLUDE_DELETED_ITEMS);
     }
 
+    public static boolean getMavenPrefOptionStatus(String prefName) {
+        String projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
+        ProjectPreferenceManager manager = getPreferenceManager(projectTechName);
+        return manager.getBoolean(prefName);
+    }
+
     private static String getGroupId(String projectTechName, String baseName, Property property) {
         if (projectTechName == null) {
             projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
@@ -355,8 +361,9 @@ public class PomIdsHelper {
 
     public static boolean skipFolders() {
         String projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
-        ProjectPreferenceManager manager = getPreferenceManager(projectTechName);
-        return manager.getBoolean(MavenConstants.SKIP_FOLDERS);
+        Project project = ProjectManager.getInstance().getProjectFromProjectTechLabel(projectTechName);
+        ProjectPreferenceManager preferenceManager = new ProjectPreferenceManager(project, DesignerMavenPlugin.PLUGIN_ID, false);
+        return preferenceManager.getBoolean(MavenConstants.SKIP_FOLDERS);
     }
 
     public static boolean isValidGroupId(String text) {
@@ -392,6 +399,7 @@ public class PomIdsHelper {
                 if (PluginChecker.isTIS()) {
                     preferenceStore.setValue(MavenConstants.EXCLUDE_DELETED_ITEMS, true);
                 }
+                preferenceStore.setValue(MavenConstants.SKIP_LOOP_DEPENDENCY_CHECK, true);
             }
             preferenceManager.save();
             preferenceManagers.put(projectTechName, preferenceManager);

main/plugins/org.talend.libraries.apache.common/pom.xml

@@ -57,6 +57,16 @@
 									<artifactId>commons-pool2</artifactId>
 									<version>2.4.2</version>
 								</artifactItem>
+								<artifactItem>
+									<groupId>commons-collections</groupId>
+									<artifactId>commons-collections</artifactId>
+									<version>3.2.2</version>
+								</artifactItem>
+								<artifactItem>
+									<groupId>commons-io</groupId>
+									<artifactId>commons-io</artifactId>
+									<version>2.4</version>
+								</artifactItem>
 							</artifactItems>
 						</configuration>
 					</execution>

main/plugins/org.talend.libraries.apache.cxf/.classpath

@@ -2,28 +2,28 @@
 <classpath>
 	<classpathentry kind="src" path="src"/>
 	<classpathentry exported="true" kind="lib" path="lib/jakarta.activation-1.2.2.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-wsdl-3.3.7.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-wsdl-3.3.10.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/jakarta.activation-api-1.2.2.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/woodstox-core-6.2.1.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/jakarta.annotation-api-1.3.5.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/stax2-api-4.2.1.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/neethi-3.1.1.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/jakarta.ws.rs-api-2.1.6.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-core-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-bindings-xml-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-frontend-jaxrs-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-rs-client-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-transports-http-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-bindings-soap-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-databinding-jaxb-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-features-clustering-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-frontend-jaxws-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-frontend-simple-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-security-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-security-saml-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-ws-addr-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-ws-policy-3.3.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-ws-security-3.3.7.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-core-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-bindings-xml-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-frontend-jaxrs-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-rs-client-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-transports-http-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-bindings-soap-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-databinding-jaxb-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-features-clustering-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-frontend-jaxws-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-frontend-simple-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-security-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-security-saml-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-ws-addr-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-ws-policy-3.3.10.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-ws-security-3.3.10.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/javax.ws.rs-api-2.0-m10.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/jakarta.xml.bind-api-2.3.3.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/xmlschema-core-2.2.5.jar"/>

main/plugins/org.talend.libraries.apache.cxf/META-INF/MANIFEST.MF

@@ -8,22 +8,22 @@ Bundle-ClassPath: .,
  lib/neethi-3.1.1.jar,
  lib/stax2-api-4.2.1.jar,
  lib/javax.activation-1.2.0.jar,
- lib/cxf-core-3.3.7.jar,
- lib/cxf-rt-bindings-xml-3.3.7.jar,
- lib/cxf-rt-frontend-jaxrs-3.3.7.jar,
- lib/cxf-rt-rs-client-3.3.7.jar,
- lib/cxf-rt-transports-http-3.3.7.jar,
- lib/cxf-rt-wsdl-3.3.7.jar,
- lib/cxf-rt-bindings-soap-3.3.7.jar,
- lib/cxf-rt-databinding-jaxb-3.3.7.jar,
- lib/cxf-rt-features-clustering-3.3.7.jar,
- lib/cxf-rt-frontend-jaxws-3.3.7.jar,
- lib/cxf-rt-frontend-simple-3.3.7.jar,
- lib/cxf-rt-security-3.3.7.jar,
- lib/cxf-rt-security-saml-3.3.7.jar,
- lib/cxf-rt-ws-addr-3.3.7.jar,
- lib/cxf-rt-ws-policy-3.3.7.jar,
- lib/cxf-rt-ws-security-3.3.7.jar,
+ lib/cxf-core-3.3.10.jar,
+ lib/cxf-rt-bindings-xml-3.3.10.jar,
+ lib/cxf-rt-frontend-jaxrs-3.3.10.jar,
+ lib/cxf-rt-rs-client-3.3.10.jar,
+ lib/cxf-rt-transports-http-3.3.10.jar,
+ lib/cxf-rt-wsdl-3.3.10.jar,
+ lib/cxf-rt-bindings-soap-3.3.10.jar,
+ lib/cxf-rt-databinding-jaxb-3.3.10.jar,
+ lib/cxf-rt-features-clustering-3.3.10.jar,
+ lib/cxf-rt-frontend-jaxws-3.3.10.jar,
+ lib/cxf-rt-frontend-simple-3.3.10.jar,
+ lib/cxf-rt-security-3.3.10.jar,
+ lib/cxf-rt-security-saml-3.3.10.jar,
+ lib/cxf-rt-ws-addr-3.3.10.jar,
+ lib/cxf-rt-ws-policy-3.3.10.jar,
+ lib/cxf-rt-ws-security-3.3.10.jar,
  lib/jakarta.ws.rs-api-2.1.6.jar,
  lib/jakarta.annotation-api-1.3.5.jar,
  lib/woodstox-core-6.2.1.jar,

main/plugins/org.talend.libraries.apache.cxf/pom.xml

@@ -21,82 +21,82 @@
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-core</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-bindings-xml</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-frontend-jaxrs</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-rs-client</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-transports-http</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-wsdl</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-bindings-soap</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-databinding-jaxb</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-features-clustering</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-frontend-jaxws</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-frontend-simple</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-security</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-security-saml</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-ws-addr</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-ws-policy</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.cxf</groupId>
 			<artifactId>cxf-rt-ws-security</artifactId>
-			<version>3.3.7</version>
+			<version>3.3.10</version>
 		</dependency>
 		<dependency>
 			<groupId>com.sun.activation</groupId>

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/ModulesNeededProvider.java

@@ -540,9 +540,9 @@ public class ModulesNeededProvider {
                         Property property = findRoutinesPropery(infor.getId(), infor.getName(), routines, type);
                         if (property != null) {
                             if (((RoutineItem) property.getItem()).isBuiltIn()) {
-                                systemRoutines.add(infor.getId());
+                                systemRoutines.add(property.getId());
                             } else {
-                                userRoutines.add(infor.getId());
+                                userRoutines.add(property.getId());
                             }
                         }
                     }

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/utils/DBConnectionContextUtils.java

@@ -25,6 +25,7 @@ import org.eclipse.core.runtime.Path;
 import org.eclipse.emf.common.util.EMap;
 import org.talend.commons.ui.utils.PathUtils;
 import org.talend.core.database.EDatabaseTypeName;
+import org.talend.core.database.EImpalaDriver;
 import org.talend.core.database.conn.ConnParameterKeys;
 import org.talend.core.database.conn.DatabaseConnStrUtil;
 import org.talend.core.database.conn.template.DbConnStrForHive;
@@ -1418,8 +1419,12 @@ public final class DBConnectionContextUtils {
         // Added 20130311 TDQ-7000, when it is context mode and not general jdbc, reset the url.
         String newURL = null;
         if (EDatabaseTypeName.IMPALA.equals(EDatabaseTypeName.getTypeFromDbType(dbConn.getDatabaseType()))) {
+            String template = DbConnStrForHive.URL_HIVE_2_TEMPLATE;
+            if (EImpalaDriver.IMPALA.getName().equals(cloneConn.getParameters().get(ConnParameterKeys.IMPALA_DRIVER))) {
+                template = DbConnStrForHive.URL_IMPALA_TEMPLATE;
+            }
             newURL = DatabaseConnStrUtil.getImpalaString(cloneConn, cloneConn.getServerName(), cloneConn.getPort(),
-                    cloneConn.getSID(), DbConnStrForHive.URL_HIVE_2_TEMPLATE);
+                    cloneConn.getSID(), template);
         } else {
             newURL = DatabaseConnStrUtil.getURLString(cloneConn.getDatabaseType(), dbConn.getDbVersionString(), server, username,
                     password, port, sidOrDatabase, filePath.toLowerCase(), datasource, dbRootPath, additionParam);

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/utils/OtherConnectionContextUtils.java

@@ -139,6 +139,7 @@ public final class OtherConnectionContextUtils {
         DbSchema,
         DbUsername,
         DbPassword,
+        DbParameters,
     }
 
     /*
@@ -611,6 +612,10 @@ public final class OtherConnectionContextUtils {
                             conn.getValue(TaggedValueHelper.getValueString(ISAPConstant.PROP_DB_PASSWORD, conn), false),
                             JavaTypesManager.PASSWORD);
                     break;
+                case DbParameters:
+                    ConnectionContextHelper.createParameters(varList, paramName,
+                            TaggedValueHelper.getValueString(ISAPConstant.PROP_DB_ADDITIONAL_PROPERTIES, conn));
+                    break;
                 default:
                 }
             }
@@ -722,6 +727,10 @@ public final class OtherConnectionContextUtils {
             TaggedValueHelper.setTaggedValue(sapConn, ISAPConstant.PROP_DB_PASSWORD,
                     ContextParameterUtils.getNewScriptCode(sapBasicVarName, LANGUAGE));
             break;
+        case DbParameters:
+            TaggedValueHelper.setTaggedValue(sapConn, ISAPConstant.PROP_DB_ADDITIONAL_PROPERTIES,
+                    ContextParameterUtils.getNewScriptCode(sapBasicVarName, LANGUAGE));
+            break;
         default:
         }
     }
@@ -768,11 +777,14 @@ public final class OtherConnectionContextUtils {
                 TaggedValueHelper.getValueString(ISAPConstant.PROP_DB_USERNAME, conn)));
         String dbPassword = TalendQuoteUtils.removeQuotes(ConnectionContextHelper.getOriginalValue(contextType,
                 conn.getValue(TaggedValueHelper.getValueString(ISAPConstant.PROP_DB_PASSWORD, conn), false)));
+        String dbParameters = TalendQuoteUtils.removeQuotes(ConnectionContextHelper.getOriginalValue(contextType,
+                TaggedValueHelper.getValueString(ISAPConstant.PROP_DB_ADDITIONAL_PROPERTIES, conn)));
         TaggedValueHelper.setTaggedValue(conn, ISAPConstant.PROP_DB_HOST, dbHost);
         TaggedValueHelper.setTaggedValue(conn, ISAPConstant.PROP_DB_PORT, dbPort);
         TaggedValueHelper.setTaggedValue(conn, ISAPConstant.PROP_DB_SCHEMA, dbSchema);
         TaggedValueHelper.setTaggedValue(conn, ISAPConstant.PROP_DB_USERNAME, dbUsername);
         TaggedValueHelper.setTaggedValue(conn, ISAPConstant.PROP_DB_PASSWORD, conn.getValue(dbPassword, true));
+        TaggedValueHelper.setTaggedValue(conn, ISAPConstant.PROP_DB_ADDITIONAL_PROPERTIES, dbParameters);
     }
 
     public static SAPConnection cloneOriginalValueSAPConnection(SAPConnection fileConn, ContextType contextType) {
@@ -813,11 +825,14 @@ public final class OtherConnectionContextUtils {
                 TaggedValueHelper.getValueString(ISAPConstant.PROP_DB_USERNAME, fileConn));
         String dbPassword = ConnectionContextHelper.getOriginalValue(contextType,
                 fileConn.getValue(TaggedValueHelper.getValueString(ISAPConstant.PROP_DB_PASSWORD, fileConn), false));
+        String dbParameters = ConnectionContextHelper.getOriginalValue(contextType,
+                TaggedValueHelper.getValueString(ISAPConstant.PROP_DB_ADDITIONAL_PROPERTIES, fileConn));
         TaggedValueHelper.setTaggedValue(cloneConn, ISAPConstant.PROP_DB_HOST, dbHost);
         TaggedValueHelper.setTaggedValue(cloneConn, ISAPConstant.PROP_DB_PORT, dbPort);
         TaggedValueHelper.setTaggedValue(cloneConn, ISAPConstant.PROP_DB_SCHEMA, dbSchema);
         TaggedValueHelper.setTaggedValue(cloneConn, ISAPConstant.PROP_DB_USERNAME, dbUsername);
         TaggedValueHelper.setTaggedValue(cloneConn, ISAPConstant.PROP_DB_PASSWORD, dbPassword);
+        TaggedValueHelper.setTaggedValue(cloneConn, ISAPConstant.PROP_DB_ADDITIONAL_PROPERTIES, dbParameters);
 
         ConnectionContextHelper.cloneConnectionProperties(fileConn, cloneConn);
 

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/metadata/managment/connection/manager/ImpalaConnectionManager.java

@@ -131,11 +131,8 @@ public class ImpalaConnectionManager extends DataBaseConnectionManager {
                                     if (EImpalaDriver.HIVE2.getDisplayName().equalsIgnoreCase(driverType)) {
                                         driverClass = EImpalaDriver.HIVE2.getDriver();
                                     }
-                                    if (EImpalaDriver.IMPALA40.getDisplayName().equalsIgnoreCase(driverType)) {
-                                        driverClass = EImpalaDriver.IMPALA40.getDriver();
-                                    }
-                                    if (EImpalaDriver.IMPALA41.getDisplayName().equalsIgnoreCase(driverType)) {
-                                        driverClass = EImpalaDriver.IMPALA41.getDriver();
+                                    if (EImpalaDriver.IMPALA.getDisplayName().equalsIgnoreCase(driverType)) {
+                                        driverClass = EImpalaDriver.IMPALA.getDriver();
                                     }
                                 } else {
                                     throw new IllegalArgumentException("impala can not work with Hive1");
@@ -165,14 +162,18 @@ public class ImpalaConnectionManager extends DataBaseConnectionManager {
         newThread.start();
 
         Connection conn = null;
+        String connectionInfo = new StringBuilder().append("JDBC Uri: ").append(metadataConn.getUrl()).append("  ").toString();
         try {
             conn = futureTask.get(getDBConnectionTimeout(), TimeUnit.SECONDS);
+            if (conn == null) {
+                throw new SQLException(connectionInfo);
+            }
         } catch (TimeoutException e) {
             threadGroup.interrupt();
             addBackgroundJob(futureTask, newThread);
-            throw new SQLException(Messages.getString("ImpalaConnectionManager.getConnection.timeout"), e); //$NON-NLS-1$
+            throw new SQLException(connectionInfo + Messages.getString("ImpalaConnectionManager.getConnection.timeout"), e); //$NON-NLS-1$
         } catch (Throwable e1) {
-            throw new SQLException(e1);
+            throw new SQLException(connectionInfo, e1);
         }
         return conn;
     }

main/plugins/org.talend.model/src/main/java/org/talend/cwm/helper/TaggedValueHelper.java

@@ -198,6 +198,9 @@ public final class TaggedValueHelper {
 
     public static final String IS_SQL_ENGIN_BEFORE_CHECK = "false"; //$NON-NLS-1$
 
+    // TDQ-19030 compare null values in Redundancy Analysis
+    public static final String IS_IGNORE_NULL = "Ignore Null"; //$NON-NLS-1$
+
     private TaggedValueHelper() {
     }
 

main/plugins/org.talend.platform.logging/src/org/talend/platform/logging/Activator.java

@@ -1,7 +1,15 @@
 package org.talend.platform.logging;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
+import org.eclipse.core.runtime.Platform;
 import org.eclipse.ui.plugin.AbstractUIPlugin;
+import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleContext;
 import org.talend.utils.format.PresentableBox;
 
@@ -18,6 +26,8 @@ public class Activator extends AbstractUIPlugin {
     // The shared instance
     private static Activator plugin;
 
+    private static String version = null;// studio version
+
     /**
      * The constructor
      */
@@ -33,6 +43,7 @@ public class Activator extends AbstractUIPlugin {
     public void start(BundleContext context) throws Exception {
         super.start(context);
         plugin = this;
+        version = null;
         if (log.isInfoEnabled()) {
             Object version = getVersion();
             String mess = "Starting Talend's platform log system."; //$NON-NLS-1$
@@ -51,6 +62,7 @@ public class Activator extends AbstractUIPlugin {
      */
     public void stop(BundleContext context) throws Exception {
         plugin = null;
+        version = null;
         super.stop(context);
     }
 
@@ -64,10 +76,43 @@ public class Activator extends AbstractUIPlugin {
     }
 
     public static String getVersion() {
-        String version = System.getProperty("talend.studio.version"); //$NON-NLS-1$
-        if (version == null || "".equals(version.trim())) { //$NON-NLS-1$
-            version = (String) getDefault().getBundle().getHeaders().get(org.osgi.framework.Constants.BUNDLE_VERSION);
+        if (version == null) {
+            try {
+                File file = null;
+                try {
+                    file = new File(Platform.getInstallLocation().getDataArea(".eclipseproduct").getPath());
+                } catch (IOException e1) {
+                    //
+                }
+                Properties prop = new Properties();
+                if (file != null && file.exists()) {
+                    try (FileInputStream fis = new FileInputStream(file)) {
+                        prop.load(fis);
+                    } catch (Exception e) {
+                        //
+                    }
+                }
+                version = prop.getProperty("version");
+
+                if (StringUtils.isEmpty(version)) {
+                    version = System.getProperty("talend.studio.version"); //$NON-NLS-1$
+                }
+
+                if (StringUtils.isEmpty(version)) {
+                    Bundle b = Platform.getBundle("org.talend.commons.runtime");
+                    if (b != null) {
+                        version = b.getHeaders().get(org.osgi.framework.Constants.BUNDLE_VERSION);
+                    }
+                }
+            } catch (Exception e) {
+                //
+            }
+
+            if (version == null || "".equals(version.trim())) { //$NON-NLS-1$
+                version = (String) getDefault().getBundle().getHeaders().get(org.osgi.framework.Constants.BUNDLE_VERSION);
+            }
         }
+
         return version;
     }
 }

main/plugins/org.talend.repository.items.importexport/src/main/java/org/talend/repository/items/importexport/handlers/HandlerUtil.java

@@ -17,6 +17,8 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URL;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.Map;

main/plugins/org.talend.repository.metadata/src/main/java/org/talend/repository/ui/wizards/metadata/connection/database/DatabaseForm.java

@@ -151,9 +151,6 @@ import org.talend.repository.model.RepositoryNode;
 import org.talend.repository.ui.dialog.AProgressMonitorDialogWithCancel;
 import org.talend.utils.sql.ConnectionUtils;
 
-import com.ca.directory.jxplorer.editor.booleaneditor;
-import com.fasterxml.jackson.databind.node.BooleanNode;
-
 /**
  * @author ocarbone
  *
@@ -6440,10 +6437,16 @@ public class DatabaseForm extends AbstractForm {
                     s = template;
                 }
             } else if (isImpalaDBConnSelected()) {
+                DatabaseConnection conn = getConnection();
                 String template = DbConnStrForHive.URL_HIVE_2_TEMPLATE;
-                s = DatabaseConnStrUtil.getImpalaString(getConnection(), getConnection().getServerName(), getConnection()
-                        .getPort(), getConnection().getSID(), template);
-                getConnection().setUiSchema(getConnection().getSID());
+                ;
+                String text = impalaDriverCombo.getText();
+                if (EDatabaseTypeName.IMPALA.getDisplayName().equals(text)) {
+                    template = DbConnStrForHive.URL_IMPALA_TEMPLATE;
+                }
+                s = DatabaseConnStrUtil.getImpalaString(conn, conn.getServerName(), conn.getPort(), conn.getSID(), template);
+                conn.setUiSchema(getConnection().getSID());
+                // conn.setURL(s);
             } else {
                 EDatabaseVersion4Drivers version = EDatabaseVersion4Drivers.indexOfByVersionDisplay(versionStr);
                 if (version != null) {
@@ -8391,12 +8394,12 @@ public class DatabaseForm extends AbstractForm {
     }
 
     private void fillDefaultsWhenImpalaVersionChanged() {
+        String distribution = getConnection().getParameters().get(ConnParameterKeys.CONN_PARA_KEY_IMPALA_DISTRIBUTION);
+        if (distribution == null) {
+            return;
+        }
+        String version = getConnection().getParameters().get(ConnParameterKeys.CONN_PARA_KEY_IMPALA_VERSION);
         if (isCreation && isNeedFillDefaults()) {
-            String distribution = getConnection().getParameters().get(ConnParameterKeys.CONN_PARA_KEY_IMPALA_DISTRIBUTION);
-            String version = getConnection().getParameters().get(ConnParameterKeys.CONN_PARA_KEY_IMPALA_VERSION);
-            if (distribution == null) {
-                return;
-            }
             EDatabaseConnTemplate template = EDatabaseConnTemplate.indexOfTemplate(getConnection().getDatabaseType());
             if (template != null) {
                 portText.setText(template.getDefaultPort());
@@ -8404,6 +8407,19 @@ public class DatabaseForm extends AbstractForm {
                 sidOrDatabaseText.setText(template.getDefaultDB(null));
             }
             initImpalaInfo();
+        } else {
+            // when edit,change distribution will update impala driver
+            IHadoopDistributionService hadoopService = getHadoopDistributionService();
+            if (hadoopService != null) {
+                IHDistribution impalaDistribution = hadoopService.getImpalaDistributionManager().getDistribution(distribution,
+                        false);
+                IHDistributionVersion hdVersion = null;
+                if (impalaDistribution != null) {
+                    hdVersion = impalaDistribution.getHDVersion(version, false);
+                }
+                // updateImpalaVersionPart(impalaDistribution);
+                updateImpalaDriverAndMakeSelection(impalaDistribution, hdVersion);
+            }
         }
     }
 
@@ -8539,9 +8555,9 @@ public class DatabaseForm extends AbstractForm {
 
     protected void doImpalaDriverSelected() {
         if (!isContextMode()) {
-            modifyFieldValue();
             getConnection().getParameters().put(ConnParameterKeys.IMPALA_DRIVER,
                     EImpalaDriver.getByDisplay(impalaDriverCombo.getText()).getName());
+            modifyFieldValue();
         }
     }
 
@@ -8686,7 +8702,7 @@ public class DatabaseForm extends AbstractForm {
         String impalaDriver = conn.getParameters().get(ConnParameterKeys.IMPALA_DRIVER);
         if (impalaDriver != null) {
             EImpalaDriver driver = EImpalaDriver.getByName(impalaDriver);
-            if (driver != null) {
+            if (driver != null && Arrays.asList(impalaDriverDisplay).contains(impalaDriver)) {
                 impalaDriverCombo.setText(driver.getDisplayName());
             } else {
                 impalaDriverCombo.select(0);

main/plugins/org.talend.repository.metadata/src/main/java/org/talend/repository/ui/wizards/metadata/table/database/DatabaseTableForm.java

@@ -93,7 +93,6 @@ import org.talend.core.model.metadata.types.JavaTypesManager;
 import org.talend.core.model.metadata.types.PerlTypesManager;
 import org.talend.core.model.metadata.types.TypesManager;
 import org.talend.core.model.properties.ConnectionItem;
-import org.talend.core.model.repository.IRepositoryPrefConstants;
 import org.talend.core.repository.model.ProxyRepositoryFactory;
 import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.ui.CoreUIPlugin;
@@ -422,9 +421,8 @@ public class DatabaseTableForm extends AbstractForm {
 
         metadataEditor.setMetadataTable(metadataTable);
 
-        Boolean flag = CoreUIPlugin.getDefault().getPreferenceStore()
-                .getBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS);
-        if (!flag.booleanValue()) {
+        boolean flag = CoreRuntimePlugin.getInstance().getProjectPreferenceManager().isAllowSpecificCharacters();
+        if (!flag) {
             List<MetadataColumn> list = metadataEditor.getMetadataColumnList();
             for (MetadataColumn column : list) {
                 if (!isCnorEn(column.getLabel())) {
@@ -1219,6 +1217,11 @@ public class DatabaseTableForm extends AbstractForm {
             boolean isHive = EDatabaseTypeName.HIVE.getDisplayName().equals(metadataconnection.getDbType());
             MappingTypeRetriever mappingTypeRetriever = getMappingTypeRetriever();
             int numbOfColumn = schemaContent.get(0).length;
+            List<TdColumn> metadataColumns = new ArrayList<TdColumn>();
+            boolean isInformix = StringUtils.equals(EDatabaseTypeName.INFORMIX.getDisplayName(), metadataconnection.getDbType());
+            if (StringUtils.isNotEmpty(tableName) && isInformix) {
+                metadataColumns = ExtractMetaDataFromDataBase.returnMetadataColumnsFormTable(metadataconnection, tableName);
+            }
             for (int i = 1; i <= numbOfColumn; i++) {
                 MetadataColumn oneColum = columns.get(i - 1);
                 // get the column name from the temp file genenrated by GuessSchemaProcess.java
@@ -1236,6 +1239,16 @@ public class DatabaseTableForm extends AbstractForm {
                 if (!"".equals(schemaContent.get(3)[i - 1])) { //$NON-NLS-1$
                     oneColum.setLength(Integer.parseInt(schemaContent.get(3)[i - 1]));
                 }
+                if (isInformix) {
+                    for (TdColumn td : metadataColumns) {
+                        if (StringUtils.equals(oneColum.getLabel(), td.getName())) {
+                            oneColum.setPrecision(td.getPrecision());
+                            oneColum.setLength(td.getLength());
+                            break;
+                        }
+                    }
+                }
+
                 if (!"".equals(schemaContent.get(4)[i - 1])) { //$NON-NLS-1$
                     oneColum.setSourceType(schemaContent.get(4)[i - 1]);
                     String talendType = MetadataTalendType.getMappingTypeRetriever(tableEditorView.getCurrentDbms())

test/plugins/org.talend.core.runtime.test/src/org/talend/core/model/metadata/MetadataToolAvroHelperTest.java

@@ -39,6 +39,7 @@ import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.metadata.types.JavaTypesManager;
 import org.talend.core.model.repository.IRepositoryPrefConstants;
 import org.talend.core.prefs.ITalendCorePrefConstants;
+import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.cwm.helper.TaggedValueHelper;
 import org.talend.daikon.avro.AvroUtils;
 import org.talend.daikon.avro.SchemaConstants;
@@ -237,8 +238,7 @@ public class MetadataToolAvroHelperTest {
         metadataTable.setSourceName("table1");
         Schema avroSchema = new Schema.Parser().parse((String) schemaObj);
 
-        IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, true);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(true);
         for (Schema.Field field : avroSchema.getFields()) {
             MetadataColumn metadataColumn = MetadataToolAvroHelper.convertFromAvro(field, metadataTable);
             metadataTable.getColumns().add(metadataColumn);
@@ -321,7 +321,7 @@ public class MetadataToolAvroHelperTest {
             }
         }
 
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(false);
         metadataTable = ConnectionFactory.eINSTANCE.createMetadataTable();
         metadataTable.setId("123456789");
         metadataTable.setName("table1");
@@ -356,8 +356,7 @@ public class MetadataToolAvroHelperTest {
         metadataTable.setSourceName("table1");
         Schema avroSchema = new Schema.Parser().parse((String) schemaObj);
 
-        IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, true);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(true);
         for (Schema.Field field : avroSchema.getFields()) {
             MetadataColumn metadataColumn = MetadataToolAvroHelper.convertFromAvro(field, metadataTable);
             metadataTable.getColumns().add(metadataColumn);
@@ -366,7 +365,7 @@ public class MetadataToolAvroHelperTest {
         assertTrue(metadataTable.getColumns().get(0).getLabel().equals("主鍵"));
         assertTrue(metadataTable.getColumns().get(1).getLabel().equals("名前"));
 
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(false);
         metadataTable = ConnectionFactory.eINSTANCE.createMetadataTable();
         metadataTable.setId("123456789");
         metadataTable.setName("table1");
@@ -453,8 +452,7 @@ public class MetadataToolAvroHelperTest {
         creatMetadataColumn.setTalendType("id_String");
         metadataTable.getColumns().add(creatMetadataColumn);
 
-        IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, true);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(true);
         org.apache.avro.Schema schema =MetadataToolAvroHelper.convertToAvro(metadataTable);
         String s = "{\"type\":\"record\",\"name\":\"table1\",\"fields\":["
                 + "{\"name\":\"_long\",\"type\":[\"string\",\"null\"],\"AVRO_TECHNICAL_KEY\":\"long\",\"di.column.talendType\":\"id_String\",\"talend.field.pattern\":\"\",\"di.table.label\":\"_long\",\"talend.field.precision\":\"0\",\"di.table.comment\":\"\",\"di.column.id\":\"111111\",\"talend.field.dbColumnName\":\"long\",\"di.column.isNullable\":\"true\",\"talend.field.length\":\"0\",\"di.column.relationshipType\":\"\",\"di.column.originalLength\":\"0\",\"di.column.relatedEntity\":\"\"},"
@@ -467,7 +465,7 @@ public class MetadataToolAvroHelperTest {
                 + "\"di.table.comment\":\"\",\"di.table.name\":\"table1\",\"di.table.label\":\"table1\"}";
         assertTrue(schema.toString().equals(s));
 
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(false);
         schema =MetadataToolAvroHelper.convertToAvro(metadataTable);
         s = "{\"type\":\"record\",\"name\":\"table1\",\"fields\":["
            + "{\"name\":\"_long\",\"type\":[\"string\",\"null\"],\"AVRO_TECHNICAL_KEY\":\"long\",\"di.column.talendType\":\"id_String\",\"talend.field.pattern\":\"\",\"di.table.label\":\"_long\",\"talend.field.precision\":\"0\",\"di.table.comment\":\"\",\"di.column.id\":\"111111\",\"talend.field.dbColumnName\":\"long\",\"di.column.isNullable\":\"true\",\"talend.field.length\":\"0\",\"di.column.relationshipType\":\"\",\"di.column.originalLength\":\"0\",\"di.column.relatedEntity\":\"\"},"

test/plugins/org.talend.core.runtime.test/src/org/talend/core/model/metadata/MetadataToolHelperTest.java

@@ -241,8 +241,7 @@ public class MetadataToolHelperTest {
      */
     @Test
     public void testValidateColumnName() {
-        IEclipsePreferences preferences = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        preferences.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(false);
 
         String columnName = MetadataToolHelper.validateColumnName("public", 0);
         assertEquals(columnName, "Column0");
@@ -274,7 +273,7 @@ public class MetadataToolHelperTest {
         columnName = MetadataToolHelper.validateColumnName("你好", 0);
         assertEquals("Column0", columnName);
 
-        preferences.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, true);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(true);
 
         columnName = MetadataToolHelper.validateColumnName("你好", 0);
         assertEquals("你好", columnName);
@@ -288,8 +287,7 @@ public class MetadataToolHelperTest {
      */
     @Test
     public void testValidateTableName() {
-        IEclipsePreferences preferences = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        preferences.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(false);
 
         String tableName = "public";
         tableName = MetadataToolHelper.validateTableName(tableName);
@@ -323,7 +321,7 @@ public class MetadataToolHelperTest {
         tableName = MetadataToolHelper.validateTableName(tableName);
         assertEquals(tableName, "t_ht01______2017");
 
-        preferences.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, true);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(true);
 
         tableName = "t_ht01_处理日期_2017";
         tableName = MetadataToolHelper.validateTableName(tableName);
@@ -647,8 +645,7 @@ public class MetadataToolHelperTest {
         creatMetadataColumn.getTaggedValue().add(tv);
         inputTable.getColumns().add(creatMetadataColumn);
 
-        IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, true);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(true);
         IMetadataTable targetTable = MetadataToolHelper.convert(inputTable);
         assertTrue(targetTable.getListColumns().get(0).getLabel().equals("_long"));
         assertTrue(targetTable.getListColumns().get(0).getOriginalDbColumnName().equals("long"));

test/plugins/org.talend.core.runtime.test/src/org/talend/core/model/utils/ContextParameterUtilsTest.java

@@ -32,6 +32,7 @@ import org.talend.core.model.process.IContext;
 import org.talend.core.model.process.IContextParameter;
 import org.talend.core.model.repository.IRepositoryPrefConstants;
 import org.talend.core.prefs.ITalendCorePrefConstants;
+import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextType;
 import org.talend.designer.core.model.utils.emf.talendfile.TalendFileFactory;
@@ -104,8 +105,7 @@ public class ContextParameterUtilsTest {
 
     @Test
     public void testGetVariableFromCode4String() {
-        IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, true);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(true);
         Assert.assertNull(ContextParameterUtils.getVariableFromCode(""));
         Assert.assertNull(ContextParameterUtils.getVariableFromCode("abc"));
         Assert.assertNull(ContextParameterUtils.getVariableFromCode("123"));
@@ -126,7 +126,7 @@ public class ContextParameterUtilsTest {
         Assert.assertEquals("Română", ContextParameterUtils.getVariableFromCode("context.Română"));
         Assert.assertEquals("русский", ContextParameterUtils.getVariableFromCode("context.русский"));
 
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(false);
         Assert.assertNull(ContextParameterUtils.getVariableFromCode("context.汉语"));
         Assert.assertNull(ContextParameterUtils.getVariableFromCode("context.日本語"));
         Assert.assertNull(ContextParameterUtils.getVariableFromCode("context.Ελληνική"));
@@ -138,8 +138,7 @@ public class ContextParameterUtilsTest {
 
     @Test
     public void testGetVariableFromCode4Context() {
-        IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, true);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(true);
         String var = ContextParameterUtils.getVariableFromCode("context.abc");
         Assert.assertEquals("abc", var);
 
@@ -275,7 +274,7 @@ public class ContextParameterUtilsTest {
         var = ContextParameterUtils.getVariableFromCode("context.русский-123");
         Assert.assertEquals("русский", var);
 
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(false);
         Assert.assertNull(ContextParameterUtils.getVariableFromCode("context.マイSQL"));
         Assert.assertNull(ContextParameterUtils.getVariableFromCode("context.汉语"));
         Assert.assertNull(ContextParameterUtils.getVariableFromCode("context.Ελληνική"));
@@ -311,8 +310,7 @@ public class ContextParameterUtilsTest {
 
     @Test
     public void testIsValidParameterName() {
-        IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, true);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(true);
         assertTrue(ContextParameterUtils.isValidParameterName("abc"));
         assertTrue(ContextParameterUtils.isValidParameterName("abc123"));
         assertTrue(ContextParameterUtils.isValidParameterName("abc_123"));
@@ -330,7 +328,7 @@ public class ContextParameterUtilsTest {
         assertTrue(ContextParameterUtils.isValidParameterName("Română"));
         assertTrue(ContextParameterUtils.isValidParameterName("русский"));
 
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(false);
         assertFalse(ContextParameterUtils.isValidParameterName("中文"));
         assertFalse(ContextParameterUtils.isValidParameterName("日本語"));
         assertFalse(ContextParameterUtils.isValidParameterName("Ελληνική"));

test/plugins/org.talend.core.test/src/org/talend/designer/runprocess/ProcessorUtilitiesTest.java

@@ -15,7 +15,6 @@ package org.talend.designer.runprocess;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.anyObject;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -30,7 +29,6 @@ import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -40,11 +38,9 @@ import java.util.jar.JarInputStream;
 import java.util.stream.Collectors;
 
 import org.apache.logging.log4j.Level;
-import org.eclipse.core.runtime.Path;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
-import org.talend.commons.exception.PersistenceException;
 import org.talend.core.model.components.EComponentType;
 import org.talend.core.model.components.IComponent;
 import org.talend.core.model.general.ModuleNeeded;
@@ -56,21 +52,8 @@ import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.INode;
 import org.talend.core.model.process.IProcess;
 import org.talend.core.model.process.JobInfo;
-import org.talend.core.model.process.ProcessUtils;
-import org.talend.core.model.properties.ProcessItem;
-import org.talend.core.model.properties.PropertiesFactory;
-import org.talend.core.model.properties.Property;
-import org.talend.core.model.relationship.Relation;
-import org.talend.core.model.relationship.RelationshipItemBuilder;
-import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.runprocess.shadow.ObjectElementParameter;
-import org.talend.core.repository.model.ProxyRepositoryFactory;
 import org.talend.core.runtime.process.LastGenerationInfo;
-import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
-import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
-import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
-import org.talend.designer.core.model.utils.emf.talendfile.TalendFileFactory;
-import org.talend.repository.ProjectManager;
 
 /**
  * DOC ggu class global comment. Detailled comment
@@ -448,102 +431,6 @@ public class ProcessorUtilitiesTest {
         assertTrue("DB node in job. shoud be metadata dynamic", ProcessorUtilities.hasMetadataDynamic(proc, null));
     }
 
-    @Test
-    public void testCheckLoopDependencies() {
-        String projectTecLabel = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
-        ProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
-        RelationshipItemBuilder relationshipItemBuilder = RelationshipItemBuilder.getInstance();
-
-        ProcessItem item = prepareProcessItem(factory.getNextId(), "test", "0.1");
-        ProcessItem item1 = prepareProcessItem(factory.getNextId(), "test1", "0.1");
-        ProcessItem item2 = prepareProcessItem(factory.getNextId(), "test2", "0.1");
-        prepareTRunjobNode(item, projectTecLabel, item1.getProperty().getId(), RelationshipItemBuilder.LATEST_VERSION);
-        prepareTRunjobNode(item1, projectTecLabel, item2.getProperty().getId(), RelationshipItemBuilder.LATEST_VERSION);
-        prepareTRunjobNode(item2, projectTecLabel, item.getProperty().getId(), RelationshipItemBuilder.LATEST_VERSION);
-
-        IRepositoryViewObject repositoryObject = null;
-        IRepositoryViewObject repositoryObject1 = null;
-        IRepositoryViewObject repositoryObject2 = null;
-        try {
-            factory.create(item, new Path(""));
-            factory.create(item1, new Path(""));
-            factory.create(item2, new Path(""));
-
-            repositoryObject = factory.getSpecificVersion(item.getProperty().getId(), item.getProperty().getVersion(), true);
-            repositoryObject1 = factory.getSpecificVersion(item1.getProperty().getId(), item1.getProperty().getVersion(), true);
-            repositoryObject2 = factory.getSpecificVersion(item2.getProperty().getId(), item2.getProperty().getVersion(), true);
-            relationshipItemBuilder.addOrUpdateItem(repositoryObject.getProperty().getItem());
-            relationshipItemBuilder.addOrUpdateItem(repositoryObject1.getProperty().getItem());
-            relationshipItemBuilder.addOrUpdateItem(repositoryObject2.getProperty().getItem());
-
-            Relation mainRelation = new Relation();
-            mainRelation.setId(repositoryObject.getProperty().getId());
-            mainRelation.setVersion(repositoryObject.getProperty().getVersion());
-            mainRelation.setType(RelationshipItemBuilder.JOB_RELATION);
-            // job-->job1-->job2-->job hasLoop==true
-            boolean hasLoop = ProcessorUtilities.checkLoopDependencies(mainRelation, new HashMap<String, String>());
-            assertTrue(hasLoop);
-
-            // job-->job1-->job2-->job(tRunjob deactivate)
-            ProcessItem jobItem2 = (ProcessItem) repositoryObject2.getProperty().getItem();
-            for (Object nodeObject : jobItem2.getProcess().getNode()) {
-                NodeType node = (NodeType) nodeObject;
-                if (!node.getComponentName().equals("tRunJob")) { // $NON-NLS-1$
-                    continue;
-                }
-                ElementParameterType actParam = TalendFileFactory.eINSTANCE.createElementParameterType();
-                actParam.setField("CHECK");
-                actParam.setName("ACTIVATE");
-                actParam.setValue("false");
-                node.getElementParameter().add(actParam);
-            }
-            factory.save(jobItem2, false);
-            hasLoop = ProcessorUtilities.checkLoopDependencies(mainRelation, new HashMap<String, String>());
-            assertFalse(hasLoop);
-
-        } catch (Exception e) {
-            e.printStackTrace();
-            fail("Test CheckLoopDependencies failure.");
-        } finally {
-            try {
-                factory.deleteObjectPhysical(repositoryObject);
-                factory.deleteObjectPhysical(repositoryObject1);
-                factory.deleteObjectPhysical(repositoryObject2);
-            } catch (PersistenceException e) {
-                e.printStackTrace();
-                fail("Test CheckLoopDependencies failure.");
-            }
-        }
-    }
-
-    private ProcessItem prepareProcessItem(String id, String label, String version) {
-        Property property = PropertiesFactory.eINSTANCE.createProperty();
-        ProcessItem item = PropertiesFactory.eINSTANCE.createProcessItem();
-        ProcessType process = TalendFileFactory.eINSTANCE.createProcessType();
-        item.setProperty(property);
-        item.setProcess(process);
-        property.setId(id);
-        property.setLabel(label);
-        property.setVersion(version);
-        return item;
-    }
-
-    private void prepareTRunjobNode(ProcessItem item, String projectLabel, String subjobId, String subjobVersion) {
-        NodeType node = TalendFileFactory.eINSTANCE.createNodeType();
-        node.setComponentName("tRunJob");
-        item.getProcess().getNode().add(node);
-        ElementParameterType versionParam = TalendFileFactory.eINSTANCE.createElementParameterType();
-        versionParam.setField("TECHNICAL");
-        versionParam.setName("PROCESS:PROCESS_TYPE_VERSION");
-        versionParam.setValue(subjobVersion);
-        node.getElementParameter().add(versionParam);
-        ElementParameterType jobIdParam = TalendFileFactory.eINSTANCE.createElementParameterType();
-        jobIdParam.setField("TECHNICAL");
-        jobIdParam.setName("PROCESS:PROCESS_TYPE_PROCESS");
-        jobIdParam.setValue(projectLabel + ProcessUtils.PROJECT_ID_SEPARATOR + subjobId);
-        node.getElementParameter().add(jobIdParam);
-    }
-    
     @Ignore
     @Test
     public void testWriteLog4j2ConfToFile() throws IOException {

test/plugins/org.talend.metadata.managment.test/src/org/talend/core/model/metadata/builder/ConvertionHelperTest.java

@@ -317,8 +317,7 @@ public class ConvertionHelperTest {
         newColumn.getAdditionalField().put("AVRO_TECHNICAL_KEY", "TEST1");
         source.getListColumns().add(newColumn);
 
-        IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, true);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(true);
         org.talend.core.model.metadata.builder.connection.MetadataTable metadataTable = ConvertionHelper.convert(source);
         assertTrue(metadataTable.getColumns().get(0).getLabel().equals("_long"));
         assertTrue(metadataTable.getColumns().get(0).getName().equals("long"));
@@ -466,8 +465,7 @@ public class ConvertionHelperTest {
         creatMetadataColumn.getTaggedValue().add(tv);
         inputTable.getColumns().add(creatMetadataColumn);
 
-        IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
-        coreUIPluginNode.putBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, true);
+        CoreRuntimePlugin.getInstance().getProjectPreferenceManager().setAllowSpecificCharacters(true);
         IMetadataTable targetTable = MetadataToolHelper.convert(inputTable);
         assertTrue(targetTable.getListColumns().get(0).getLabel().equals("_long"));
         assertTrue(targetTable.getListColumns().get(0).getOriginalDbColumnName().equals("long"));