bugfix(TUP-24978) can not download ojdbc8-12.2.0.1.jar (#2793)

* bugfix(TUP-24978) Download jar

* bugfix(TUP-24986) For maven artifact resolver, rever to http transporter

* bugfix(TUP-24978) Remove unused import

main/features/org.talend.utils.feature/feature.xml

@@ -20,8 +20,10 @@
     <import plugin="org.apache.servicemix.bundles.avro" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.junit" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.slf4j.api" version="0.0.0" match="greaterOrEqual"/>
+    <import plugin="org.apache.commons.configuration" version="2.0.0" match="greaterOrEqual"/>
   </requires>  
   <plugin id="org.talend.daikon" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
   <plugin id="org.talend.daikon.exception" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
+  <plugin id="org.talend.daikon.crypto.utils" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
   <plugin id="org.talend.utils" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
 </feature>

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/PasswordEncryptUtil.java

@@ -21,29 +21,25 @@ import javax.crypto.SecretKeyFactory;
 import javax.crypto.spec.DESKeySpec;
 
 import org.apache.commons.codec.binary.Base64;
-import org.talend.utils.security.AESEncryption;
+import org.talend.utils.security.StudioEncryption;
 
 /**
  * DOC chuang class global comment. Detailled comment
  */
 public class PasswordEncryptUtil {
 
-    public static String ENCRYPT_KEY = "Encrypt"; //$NON-NLS-1$
+    public static final String ENCRYPT_KEY = "Encrypt"; //$NON-NLS-1$
 
-    private static String rawKey = "Talend-Key"; //$NON-NLS-1$
-
-    public static String PREFIX_PASSWORD = "ENC:["; //$NON-NLS-1$
-
-    public static String POSTFIX_PASSWORD = "]"; //$NON-NLS-1$
+    private static final String RAWKEY = "Talend-Key"; //$NON-NLS-1$
 
     private static SecretKey key = null;
 
-    private static SecureRandom secureRandom = new SecureRandom();
+    private static final SecureRandom SECURERANDOM = new SecureRandom();
 
     private static SecretKey getSecretKey() throws Exception {
         if (key == null) {
 
-            byte rawKeyData[] = rawKey.getBytes();
+            byte rawKeyData[] = RAWKEY.getBytes();
             DESKeySpec dks = new DESKeySpec(rawKeyData);
             SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("DES"); //$NON-NLS-1$
             key = keyFactory.generateSecret(dks);
@@ -65,7 +61,7 @@ public class PasswordEncryptUtil {
 
         SecretKey key = getSecretKey();
         Cipher c = Cipher.getInstance("DES"); //$NON-NLS-1$
-        c.init(Cipher.ENCRYPT_MODE, key, secureRandom);
+        c.init(Cipher.ENCRYPT_MODE, key, SECURERANDOM);
         byte[] cipherByte = c.doFinal(input.getBytes());
         String dec = new String(Base64.encodeBase64(cipherByte));
         return dec;
@@ -85,7 +81,7 @@ public class PasswordEncryptUtil {
         byte[] dec = Base64.decodeBase64(input.getBytes());
         SecretKey key = getSecretKey();
         Cipher c = Cipher.getInstance("DES"); //$NON-NLS-1$
-        c.init(Cipher.DECRYPT_MODE, key, secureRandom);
+        c.init(Cipher.DECRYPT_MODE, key, SECURERANDOM);
         byte[] clearByte = c.doFinal(dec);
         return new String(clearByte);
     }
@@ -99,7 +95,7 @@ public class PasswordEncryptUtil {
         if (input == null) {
             return input;
         }
-        return PREFIX_PASSWORD + AESEncryption.encryptPassword(input) + POSTFIX_PASSWORD;
+        return StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.ROUTINE).encrypt(input);
     }
 
     /**

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/VersionUtils.java

@@ -19,6 +19,7 @@ import java.io.InputStream;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.Properties;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.StringUtils;
@@ -192,6 +193,27 @@ public class VersionUtils {
         return talendVersion;
     }
 
+    public static String getTalendPureVersion(String fullProductVersion) {
+        String version = fullProductVersion;
+        String[] splitStr = fullProductVersion.split("-"); //$NON-NLS-1$
+        Pattern pattern = Pattern.compile("((\\d+\\.){2}\\d.*)"); //$NON-NLS-1$
+        StringBuffer versionStr = new StringBuffer();
+        boolean find = false;
+        for (String str : splitStr) {
+            if (find) {
+                versionStr.append("-").append(str); //$NON-NLS-1$
+            }else {
+                Matcher matcher = pattern.matcher(str);
+                if (matcher.find()) {
+                    find = true;
+                    versionStr.append(str); // $NON-NLS-1$
+                }
+            }
+        }
+
+        return getTalendVersion(versionStr.toString());
+    }
+
     public static String getTalendVersion(String productVersion) {
         try {
             org.osgi.framework.Version v = new org.osgi.framework.Version(productVersion);

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/linking/TreeToTablesLinker.java

@@ -33,6 +33,7 @@ import org.eclipse.swt.widgets.TreeItem;
 import org.talend.commons.ui.runtime.i18n.Messages;
 import org.talend.commons.ui.runtime.utils.TableUtils;
 import org.talend.commons.ui.runtime.ws.WindowSystem;
+import org.talend.commons.ui.swt.advanced.dataeditor.AbstractDataTableEditorView;
 import org.talend.commons.ui.swt.drawing.background.IBackgroundRefresher;
 import org.talend.commons.ui.swt.drawing.background.IBgDrawableComposite;
 import org.talend.commons.ui.swt.drawing.link.BezierHorizontalLink;
@@ -464,4 +465,19 @@ public class TreeToTablesLinker<D1, D2> extends BgDrawableComposite implements I
         }
     }
 
+    protected <B> void loadItemDataForLazyLoad(AbstractDataTableEditorView<B> tableEditorView) {
+        if (!tableEditorView.getTableViewerCreator().isLazyLoad()) {
+            return;
+        }
+        List<B> beansList = tableEditorView.getExtendedTableModel().getBeansList();
+        Table table = tableEditorView.getTable();
+        for (TableItem tableItem : table.getItems()) {
+            if (tableItem.getData() == null) {
+                int index = table.indexOf(tableItem);
+                B schemaTarget = beansList.get(index);
+                tableItem.setData(schemaTarget);
+            }
+        }
+    }
+
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/logintask/SyncLibrariesLoginTask.java

@@ -44,26 +44,27 @@ public class SyncLibrariesLoginTask extends AbstractLoginTask implements IRunnab
         ProxyRepositoryFactory.getInstance().executeRepositoryWorkUnit(
                 new RepositoryWorkUnit<Void>(Messages.getString("SyncLibrariesLoginTask.createStatsLogAndImplicitParamter")) {
 
-            @Override
-            protected void run() throws LoginException, PersistenceException {
-                try {
-                    IWorkspace workspace = ResourcesPlugin.getWorkspace();
-                    IResourceRuleFactory ruleFactory = workspace.getRuleFactory();
-                    ProjectManager projectManager = ProjectManager.getInstance();
-                    ISchedulingRule refreshRule = ruleFactory
-                            .refreshRule(projectManager.getResourceProject(getProject().getEmfProject()));
-                    workspace.run(new IWorkspaceRunnable() {
+                    @Override
+                    protected void run() throws LoginException, PersistenceException {
+                        try {
+                            IWorkspace workspace = ResourcesPlugin.getWorkspace();
+                            IResourceRuleFactory ruleFactory = workspace.getRuleFactory();
+                            ProjectManager projectManager = ProjectManager.getInstance();
+                            ISchedulingRule refreshRule = ruleFactory.refreshRule(
+                                    projectManager.getResourceProject(projectManager.getCurrentProject().getEmfProject()));
+                            workspace.run(new IWorkspaceRunnable() {
 
-                        @Override
-                        public void run(IProgressMonitor monitor) throws CoreException {
-                            coreService.createStatsLogAndImplicitParamter(ProjectManager.getInstance().getCurrentProject());
+                                @Override
+                                public void run(IProgressMonitor monitor) throws CoreException {
+                                    coreService
+                                            .createStatsLogAndImplicitParamter(ProjectManager.getInstance().getCurrentProject());
+                                }
+                            }, refreshRule, IWorkspace.AVOID_UPDATE, monitor);
+                        } catch (Exception e) {
+                            ExceptionHandler.process(e);
                         }
-                    }, refreshRule, IWorkspace.AVOID_UPDATE, monitor);
-                } catch (Exception e) {
-                    ExceptionHandler.process(e);
-                }
-            }
-        });
+                    }
+                });
     }
 
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ILibraryManagerService.java

@@ -89,6 +89,8 @@ public interface ILibraryManagerService extends IService {
      * @return
      */
     public boolean retrieve(String jarNeeded, String pathToStore, IProgressMonitor... monitorWrap);
+    
+    public boolean retrieve(String jarNeeded, String jarURL, String pathToStore, IProgressMonitor... monitorWrap);
 
     public boolean retrieve(String jarNeeded, String pathToStore, boolean showDialog, IProgressMonitor... monitorWrap);
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ITDQRepositoryService.java

@@ -210,4 +210,9 @@ public interface ITDQRepositoryService extends IService {
      * @param ruManager: RepositoryUpdateManager
      */
     void updateAllContextInAnalysisAndReport(RepositoryUpdateManager ruManager, Object parameter, boolean isUpdated);
+
+    /**
+     * @param chooseContext the context name which want to swtich
+     */
+    void popupSwitchContextFailedMessage(String chooseContext);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/version/EDatabaseVersion4Drivers.java

@@ -164,11 +164,11 @@ public enum EDatabaseVersion4Drivers {
     MAPRDB(new DbVersion4Drivers(EDatabaseTypeName.MAPRDB, new String[] {})),
 
     REDSHIFT(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT, "redshift", "REDSHIFT", //$NON-NLS-1$ //$NON-NLS-2$
-            "redshift-jdbc42-no-awssdk-1.2.20.1043.jar")), //$NON-NLS-1$
+            "redshift-jdbc42-no-awssdk-1.2.32.1056.jar")), //$NON-NLS-1$
     REDSHIFT_SSO(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT_SSO, "redshift sso", "REDSHIFT_SSO", //$NON-NLS-1$ //$NON-NLS-2$
-            new String[] { "redshift-jdbc42-no-awssdk-1.2.20.1043.jar", "aws-java-sdk-1.11.406.jar", "jackson-core-2.9.5.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-                    "jackson-databind-2.9.5.jar", "jackson-annotations-2.9.0.jar", "httpcore-4.4.9.jar", "httpclient-4.5.5.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$//$NON-NLS-4$
-                    "joda-time-2.8.1.jar", "commons-logging-1.1.3.jar" })), //$NON-NLS-1$ //$NON-NLS-2$
+            new String[] { "redshift-jdbc42-no-awssdk-1.2.32.1056.jar", "aws-java-sdk-1.11.406.jar", "jackson-core-2.9.9.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+                    "jackson-databind-2.9.9.jar", "jackson-annotations-2.9.0.jar", "httpcore-4.4.9.jar", "httpclient-4.5.5.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$//$NON-NLS-4$
+                    "joda-time-2.8.1.jar", "commons-logging-1.1.3.jar", "commons-codec-1.6.jar" })), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
 
     AMAZON_AURORA(new DbVersion4Drivers(EDatabaseTypeName.AMAZON_AURORA, "mysql-connector-java-5.1.30-bin.jar")); //$NON-NLS-1$
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/components/IComponent.java

@@ -60,6 +60,15 @@ public interface IComponent {
 
     public String getOriginalName();
 
+    /**
+     * Only for component display (palette,search)
+     * 
+     * @return
+     */
+    default public String getDisplayName() {
+        return getName();
+    }
+
     public String getLongName();
 
     public String getOriginalFamilyName();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/context/JobContext.java

@@ -98,6 +98,15 @@ public class JobContext implements IContext, Cloneable {
         return true;
     }
 
+    public boolean containsSameParameterIgnoreCase(String parameterName) {
+        for (IContextParameter contextParam : contextParameterList) {
+            if (contextParam.getName() != null && contextParam.getName().equalsIgnoreCase(parameterName)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      * Commented by Marvin Wang on Mar.2, 2012, the user who invokes the method should notice that if the
      * <code>JobContext</code> has more than one <code>ContextParameter</code>s which names are same, this case will

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/AbstractNode.java

@@ -39,7 +39,7 @@ public abstract class AbstractNode implements INode {
 
     private ComponentProperties componentProperties;
 
-    List<? extends IElementParameter> elementParameters;
+    List<? extends IElementParameter> elementParameters = new ArrayList<IElementParameter>();
 
     private List<? extends IConnection> outgoingConnections = new ArrayList<IConnection>();
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/IContext.java

@@ -62,4 +62,6 @@ public interface IContext {
     public IContext clone();
 
     public boolean sameAs(IContext context);
+
+    public boolean containsSameParameterIgnoreCase(String parameterName);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ProcessUtils.java

@@ -25,6 +25,7 @@ import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.utils.VersionUtils;
 import org.talend.core.GlobalServiceRegister;
+import org.talend.core.IESBService;
 import org.talend.core.ITDQItemService;
 import org.talend.core.PluginChecker;
 import org.talend.core.hadoop.IHadoopClusterService;
@@ -33,7 +34,6 @@ import org.talend.core.model.general.Project;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.ProcessItem;
-import org.talend.core.model.properties.ProjectReference;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.properties.SQLPatternItem;
 import org.talend.core.model.relationship.Relation;
@@ -904,27 +904,7 @@ public final class ProcessUtils {
                 }
             }
         }
-
-        if (needBeans && GlobalServiceRegister.getDefault().isServiceRegistered(IProxyRepositoryService.class)) {
-            IProxyRepositoryService service = (IProxyRepositoryService) GlobalServiceRegister.getDefault()
-                    .getService(IProxyRepositoryService.class);
-            ERepositoryObjectType beansType = ERepositoryObjectType.valueOf("BEANS"); //$NON-NLS-1$
-            try {
-                IProxyRepositoryFactory factory = service.getProxyRepositoryFactory();
-                List<IRepositoryViewObject> all = factory.getAll(project, beansType);
-                List<ProjectReference> references = ProjectManager.getInstance().getCurrentProject()
-                        .getProjectReferenceList(true);
-                for (ProjectReference ref : references) {
-                    all.addAll(factory.getAll(new Project(ref.getReferencedProject()), beansType));
-                }
-                if (!all.isEmpty()) { // has bean
-                    return true;
-                }
-            } catch (PersistenceException e) {
-                ExceptionHandler.process(e);
-            }
-        }
-        return false;
+        return needBeans && GlobalServiceRegister.getDefault().isServiceRegistered(IESBService.class);
     }
 
     public static boolean isRequiredPigUDFs(IProcess process) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/repository/ERepositoryObjectType.java

@@ -575,6 +575,8 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
     public final static ERepositoryObjectType METADATA_SAP_BW_INFOOBJECT = ERepositoryObjectType
             .valueOf("METADATA_SAP_BW_INFOOBJECT"); //$NON-NLS-1$
 
+    public final static ERepositoryObjectType JDBC = ERepositoryObjectType.valueOf("JDBC"); //$NON-NLS-1$
+
     private static Map<String, ERepositoryObjectType> typeCacheById = new HashMap<String, ERepositoryObjectType>();
 
     ERepositoryObjectType(String key, String folder, String type, boolean isStaticNode, int ordinal, String[] products,

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/NodeUtil.java

@@ -684,16 +684,15 @@ public class NodeUtil {
             }
             List<? extends IConnection> listInConns = node.getIncomingConnections();
             if (listInConns != null && listInConns.size() > 0) {
-                String retResult = getPrivateConnClassName(listInConns.get(0));
-                if (retResult == null) {
-                    return conn.getName();
-                } else {
-                    return retResult;
+                for (IConnection connection : listInConns) {
+                    if (EConnectionType.FLOW_REF != connection.getLineStyle()) {
+                        String retResult = getPrivateConnClassName(connection);
+                        return retResult != null ? retResult : conn.getName();
+                    }
                 }
             }
         }
         return null;
-
     }
 
     /**

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/ParameterValueUtil.java

@@ -41,11 +41,11 @@ import org.talend.core.model.process.EParameterFieldType;
 import org.talend.core.model.process.IContextParameter;
 import org.talend.core.model.process.IElementParameter;
 import org.talend.core.utils.TalendQuoteUtils;
-import org.talend.daikon.security.CryptoHelper;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.ElementValueType;
 import org.talend.repository.ProjectManager;
+import org.talend.utils.security.StudioEncryption;
 
 /**
  * cli class global comment. Detailled comment
@@ -842,7 +842,8 @@ public final class ParameterValueUtil {
         if (contextParam != null) {
             String docValue = contextParam.getValue();
             if (docValue != null) {
-                String encryptValue = CryptoHelper.getDefault().encrypt(docValue);
+                String encryptValue = StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM)
+                        .encrypt(docValue);
                 if (encryptValue != null) {
                     return encryptValue;
                 }
@@ -878,7 +879,8 @@ public final class ParameterValueUtil {
         if (param != null) {
             Object docValue = param.getValue();
             if (docValue != null && docValue instanceof String) {
-                String encryptValue = CryptoHelper.getDefault().encrypt(docValue.toString());
+                String encryptValue = StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM)
+                        .encrypt(docValue.toString());
                 if (encryptValue != null) {
                     return encryptValue;
                 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/ResolverExceptionHandler.java

@@ -0,0 +1,33 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.nexus;
+
+import java.io.IOException;
+
+public class ResolverExceptionHandler {
+
+    public static IOException hideCredential(IOException e) {
+        // hide the user/password in the error
+        String regex = "\\://(.+)\\:(.+)@"; //$NON-NLS-1$
+        String message = e.getMessage();
+        message = message.replaceAll(regex, "://"); //$NON-NLS-1$
+        Exception cause = null;
+        if (e.getCause() != null) {
+            String causeMessage = e.getCause().getMessage();
+            causeMessage = causeMessage.replaceAll(regex, "://"); //$NON-NLS-1$
+            cause = new Exception(causeMessage);
+        }
+        return new IOException(message, cause);
+    }
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/TalendMavenResolver.java

@@ -101,8 +101,12 @@ public class TalendMavenResolver {
 
     }
 
-    public static File resolve(String mvnUri) throws IOException, RuntimeException {
-        return getMavenResolver().resolve(mvnUri);
+    public static File resolve(String mvnUri) throws IOException {
+        try {
+            return getMavenResolver().resolve(mvnUri);
+        } catch (IOException e) {
+            throw ResolverExceptionHandler.hideCredential(e);
+        }
     }
 
     public static void upload(String groupId, String artifactId, String classifier, String extension, String version,
@@ -110,7 +114,11 @@ public class TalendMavenResolver {
         getMavenResolver().upload(groupId, artifactId, classifier, extension, version, artifact);
     }
 
-    public static MavenResolver getMavenResolver() throws RuntimeException {
+    public static void initMavenResovler() throws RuntimeException {
+        getMavenResolver();
+    }
+
+    private static MavenResolver getMavenResolver() throws RuntimeException {
         if (mavenResolver == null) {
             final BundleContext context = CoreRuntimePlugin.getInstance().getBundle().getBundleContext();
             ServiceReference<org.ops4j.pax.url.mvn.MavenResolver> mavenResolverService = context

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenConstants.java

@@ -42,6 +42,8 @@ public interface MavenConstants {
 
     static final String POM_FILTER = "POM_FILTER";
 
+    static final String USE_PROFILE_MODULE = "USE_PROFILE_MODULE";
+
     /*
      * for lib
      */

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenUrlHelper.java

@@ -19,7 +19,7 @@ import java.net.URLDecoder;
 import org.apache.commons.lang.StringUtils;
 import org.eclipse.core.runtime.Assert;
 import org.talend.commons.exception.ExceptionHandler;
-import org.talend.daikon.security.CryptoHelper;
+import org.talend.utils.security.StudioEncryption;
 
 /**
  * DOC ggu class global comment. Detailled comment
@@ -47,8 +47,6 @@ public class MavenUrlHelper {
 
     public static final String USER_PASSWORD_SPLITER = ":";
 
-    private static CryptoHelper cryptoHelper;
-
     public static MavenArtifact parseMvnUrl(String mvnUrl) {
         return parseMvnUrl(mvnUrl, true);
     }
@@ -332,19 +330,12 @@ public class MavenUrlHelper {
         return uri;
     }
 
-    private static CryptoHelper getCryptoHelper() {
-        if (cryptoHelper == null) {
-            cryptoHelper = CryptoHelper.getDefault();
-        }
-        return cryptoHelper;
-    }
-
     public static String encryptPassword(String password) {
-        return getCryptoHelper().encrypt(password);
+        return StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM).encrypt(password);
     }
 
     public static String decryptPassword(String password) {
-        return getCryptoHelper().decrypt(password);
+        return StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM).decrypt(password);
     }
 
 	public static String generateModuleNameByMavenURI(String uri) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/IJobletProviderService.java

@@ -99,6 +99,8 @@ public interface IJobletProviderService extends IService {
     public IEditorPart openJobletItem(JobletProcessItem item);
 
     public boolean isJobletItem(Item item);
+    
+    public boolean isJobletProcess(IProcess process);
 
     public Action getMoveToJobletAction(IWorkbenchPart part, INode jobletNode, Map<INode, IConnection> nodeMap);
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/utils/StudioSSLContextProvider.java

@@ -22,8 +22,8 @@ import org.eclipse.jface.preference.IPreferenceStore;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.prefs.SSLPreferenceConstants;
 import org.talend.core.runtime.CoreRuntimePlugin;
-import org.talend.daikon.security.CryptoHelper;
 import org.talend.daikon.security.SSLContextProvider;
+import org.talend.utils.security.StudioEncryption;
 
 /**
  *
@@ -34,7 +34,7 @@ public class StudioSSLContextProvider {
 
     private static SSLContext context;
 
-    private static final IPreferenceStore store = CoreRuntimePlugin.getInstance().getCoreService().getPreferenceStore();
+    private static final IPreferenceStore STORE = CoreRuntimePlugin.getInstance().getCoreService().getPreferenceStore();
 
     public static synchronized SSLContext getContext() throws Exception {
         if (null == context) {
@@ -44,15 +44,14 @@ public class StudioSSLContextProvider {
     }
 
     public static synchronized void buildContext() throws Exception {
-        String keypath = store.getString(SSLPreferenceConstants.KEYSTORE_FILE);
-        String keypass = store.getString(SSLPreferenceConstants.KEYSTORE_PASSWORD);
-        String keytype = store.getString(SSLPreferenceConstants.KEYSTORE_TYPE);
-        String trustpath = store.getString(SSLPreferenceConstants.TRUSTSTORE_FILE);
-        String trustpass = store.getString(SSLPreferenceConstants.TRUSTSTORE_PASSWORD);
-        String trusttype = store.getString(SSLPreferenceConstants.TRUSTSTORE_TYPE);
-        CryptoHelper cryptoHelper = CryptoHelper.getDefault();
-        keypass = cryptoHelper.decrypt(keypass);
-        trustpass = cryptoHelper.decrypt(trustpass);
+        String keypath = STORE.getString(SSLPreferenceConstants.KEYSTORE_FILE);
+        String keypass = STORE.getString(SSLPreferenceConstants.KEYSTORE_PASSWORD);
+        String keytype = STORE.getString(SSLPreferenceConstants.KEYSTORE_TYPE);
+        String trustpath = STORE.getString(SSLPreferenceConstants.TRUSTSTORE_FILE);
+        String trustpass = STORE.getString(SSLPreferenceConstants.TRUSTSTORE_PASSWORD);
+        String trusttype = STORE.getString(SSLPreferenceConstants.TRUSTSTORE_TYPE);
+        keypass = StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM).decrypt(keypass);
+        trustpass = StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM).decrypt(trustpass);
         try {
             if (StringUtils.isEmpty(keypath) && StringUtils.isEmpty(trustpath)) {
                 context = null;
@@ -88,12 +87,12 @@ public class StudioSSLContextProvider {
 
     private static void changeProperty() {
         final IPreferenceStore sslStore = CoreRuntimePlugin.getInstance().getCoreService().getPreferenceStore();
-        CryptoHelper cryptoHelper = CryptoHelper.getDefault();
         String keyStore = sslStore.getString(SSLPreferenceConstants.KEYSTORE_FILE);
         if (keyStore != null && !"".equals(keyStore.trim())) {
             System.setProperty(SSLPreferenceConstants.KEYSTORE_FILE, keyStore);
             System.setProperty(SSLPreferenceConstants.KEYSTORE_PASSWORD,
-                    cryptoHelper.decrypt(sslStore.getString(SSLPreferenceConstants.KEYSTORE_PASSWORD)));
+                    StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM)
+                            .decrypt(sslStore.getString(SSLPreferenceConstants.KEYSTORE_PASSWORD)));
             System.setProperty(SSLPreferenceConstants.KEYSTORE_TYPE, sslStore.getString(SSLPreferenceConstants.KEYSTORE_TYPE));
         } else {
             System.clearProperty(SSLPreferenceConstants.KEYSTORE_FILE);
@@ -104,7 +103,8 @@ public class StudioSSLContextProvider {
         if (trustStore != null && !"".equals(trustStore.trim())) {
             System.setProperty(SSLPreferenceConstants.TRUSTSTORE_FILE, trustStore);
             System.setProperty(SSLPreferenceConstants.TRUSTSTORE_PASSWORD,
-                    cryptoHelper.decrypt(sslStore.getString(SSLPreferenceConstants.TRUSTSTORE_PASSWORD)));
+                    StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM)
+                            .decrypt(sslStore.getString(SSLPreferenceConstants.TRUSTSTORE_PASSWORD)));
             System.setProperty(SSLPreferenceConstants.TRUSTSTORE_TYPE, sslStore.getString(SSLPreferenceConstants.TRUSTSTORE_TYPE));
         } else {
             System.clearProperty(SSLPreferenceConstants.TRUSTSTORE_FILE);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/AbstractRoutineSynchronizer.java

@@ -19,7 +19,9 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
@@ -79,6 +81,8 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
     }
 
     private Collection<RoutineItem> getAll(ERepositoryObjectType type, boolean syncRef) throws SystemException {
+        // init code project
+        getRunProcessService().getTalendCodeJavaProject(type);
         // remove routine with same name in reference project
         final Map<String, RoutineItem> beansList = new HashMap<String, RoutineItem>();
         for (IRepositoryViewObject obj : getRepositoryService().getProxyRepositoryFactory().getAll(type)) {
@@ -95,27 +99,37 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
         return (IRepositoryService) GlobalServiceRegister.getDefault().getService(IRepositoryService.class);
     }
 
-    private void getReferencedProjectRoutine(final Map<String, RoutineItem> beansList, final Project project,
-            ERepositoryObjectType routineType, boolean syncRef) throws SystemException {
-        for (IRepositoryViewObject obj : getRepositoryService().getProxyRepositoryFactory().getAll(project, routineType)) {
+    private Set<IRepositoryViewObject> getReferencedProjectRoutine(final Map<String, RoutineItem> beansList,
+            final Project project, ERepositoryObjectType routineType, boolean syncRef) throws SystemException {
+        // init ref code project
+        if (syncRef) {
+            getRunProcessService().getTalendCodeJavaProject(routineType, project.getTechnicalLabel());
+        }
+        Set<IRepositoryViewObject> routines = new HashSet<>();
+        routines.addAll(getRepositoryService().getProxyRepositoryFactory().getAll(project, routineType));
+        for (IRepositoryViewObject obj : routines) {
             final String key = obj.getProperty().getLabel();
             // it does not have a routine with same name
             if (!beansList.containsKey(key)) {
                 beansList.put(key, (RoutineItem) obj.getProperty().getItem());
             }
-            if (syncRef) {
-                // sync routine
-                syncRoutine((RoutineItem) obj.getProperty().getItem(), false, true, true);
-            }
+        }
+        for (ProjectReference projectReference : project.getProjectReferenceList()) {
+            routines.addAll(getReferencedProjectRoutine(beansList, new Project(projectReference.getReferencedProject()),
+                    routineType, syncRef));
         }
         if (syncRef) {
+            routines.stream().forEach(obj -> {
+                try {
+                    syncRoutine((RoutineItem) obj.getProperty().getItem(), project.getTechnicalLabel(), true, true);
+                } catch (SystemException e) {
+                    ExceptionHandler.process(e);
+                }
+            });
             // sync system routine
             syncSystemRoutine(project);
         }
-
-        for (ProjectReference projectReference : project.getProjectReferenceList()) {
-            getReferencedProjectRoutine(beansList, new Project(projectReference.getReferencedProject()), routineType, syncRef);
-        }
+        return routines;
     }
 
     protected void syncSystemRoutine(Project project) throws SystemException {
@@ -128,24 +142,17 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
     }
 
     protected IFile getRoutineFile(RoutineItem routineItem) throws SystemException {
-        return getRoutineFile(routineItem, true);
+        return getRoutineFile(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel());
     }
 
-    protected IFile getRoutineFile(RoutineItem routineItem, boolean currentProject) throws SystemException {
-        String projectTechName;
-        if (currentProject) {
-            projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
-        } else {
-            projectTechName = ProjectManager.getInstance().getProject(routineItem).getTechnicalLabel();
-        }
+    protected IFile getRoutineFile(RoutineItem routineItem, String projectTechName) throws SystemException {
         ITalendProcessJavaProject talendProcessJavaProject = getRunProcessService()
                 .getTalendCodeJavaProject(ERepositoryObjectType.getItemType(routineItem), projectTechName);
         if (talendProcessJavaProject == null) {
             return null;
         }
         IFolder routineFolder = talendProcessJavaProject.getSrcSubFolder(null, routineItem.getPackageType());
-        IFile file = routineFolder.getFile(routineItem.getProperty().getLabel() + JavaUtils.JAVA_EXTENSION);
-        return file;
+        return routineFolder.getFile(routineItem.getProperty().getLabel() + JavaUtils.JAVA_EXTENSION);
     }
 
     private IFile getProcessFile(ProcessItem item) throws SystemException {
@@ -193,30 +200,31 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
 
     @Override
     public void syncRoutine(RoutineItem routineItem, boolean copyToTemp) throws SystemException {
-        syncRoutine(routineItem, true, copyToTemp, false);
+        syncRoutine(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), copyToTemp, false);
     }
 
     @Override
     public void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceUpdate) throws SystemException {
-        syncRoutine(routineItem, true, copyToTemp, forceUpdate);
+        syncRoutine(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), copyToTemp, forceUpdate);
     }
 
     @Override
-    public void syncRoutine(RoutineItem routineItem, boolean currentProject, boolean copyToTemp, boolean forceUpdate) throws SystemException {
+    public void syncRoutine(RoutineItem routineItem, String projectTechName, boolean copyToTemp, boolean forceUpdate)
+            throws SystemException {
         boolean needSync = false;
         if (routineItem != null) {
             if (forceUpdate || !isRoutineUptodate(routineItem)) {
                 needSync = true;
             } else {
-                IFile file = getRoutineFile(routineItem, currentProject);
+                IFile file = getRoutineFile(routineItem, projectTechName);
                 if (file != null && !file.exists()) {
                     needSync = true;
                 }
             }
         }
         if (needSync) {
-            doSyncRoutine(routineItem, currentProject, copyToTemp);
-            if (currentProject) {
+            doSyncRoutine(routineItem, projectTechName, copyToTemp);
+            if (ProjectManager.getInstance().getCurrentProject().getTechnicalLabel().equals(projectTechName)) {
                 setRoutineAsUptodate(routineItem);
             }
         }
@@ -224,14 +232,14 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
 
     public void syncRoutine(RoutineItem routineItem) throws SystemException {
         if (routineItem != null) {
-            doSyncRoutine(routineItem, true, true);
+            doSyncRoutine(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), true);
             setRoutineAsUptodate(routineItem);
         }
     }
 
-    private void doSyncRoutine(RoutineItem routineItem, boolean currentProject, boolean copyToTemp) throws SystemException {
+    private void doSyncRoutine(RoutineItem routineItem, String projectTechName, boolean copyToTemp) throws SystemException {
         try {
-            IFile file = getRoutineFile(routineItem, currentProject);
+            IFile file = getRoutineFile(routineItem, projectTechName);
             if (file == null) {
                 return;
             }
@@ -374,7 +382,7 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
     @Override
     public void syncAllBeansForLogOn() throws SystemException {
         for (RoutineItem beanItem : getBeans(true)) {
-            syncRoutine(beanItem, true, true, true);
+            syncRoutine(beanItem, true, true);
         }
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/ITalendSynchronizer.java

@@ -45,7 +45,8 @@ public interface ITalendSynchronizer {
 
     void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceUpdate) throws SystemException;
 
-    void syncRoutine(RoutineItem routineItem, boolean currentProject, boolean copyToTemp, boolean forceUpdate) throws SystemException;
+    void syncRoutine(RoutineItem routineItem, String projectTechName, boolean copyToTemp, boolean forceUpdate)
+            throws SystemException;
 
     IFile getFile(Item item) throws SystemException;
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/core/IDesignerCoreService.java

@@ -195,4 +195,7 @@ public interface IDesignerCoreService extends IService {
     public void setTACReadTimeout(int timeout);
 
     boolean isDelegateNode(INode node);
+
+    boolean isNeedContextInJar(IProcess process);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IProcessor.java

@@ -12,6 +12,7 @@
 // ============================================================================
 package org.talend.designer.runprocess;
 
+import java.util.Collection;
 import java.util.Map;
 import java.util.Set;
 
@@ -254,6 +255,9 @@ public interface IProcessor {
 
     public String[] getCommandLine(boolean needContext, boolean externalUse, int statOption, int traceOption,
             String... codeOptions);
+    
+    public String[] getCommandLine(boolean needContext, boolean externalUse, int statOption, int traceOption, boolean ignoreCustomJVMSetting,
+            String... codeOptions);
 
     public void setContext(IContext context);
 
@@ -295,6 +299,8 @@ public interface IProcessor {
     String[] getJVMArgs();
 
     Set<ModuleNeeded> getNeededModules(int options);
+    
+    public void updateModulesAfterSetLog4j(Collection<ModuleNeeded> modulesNeeded);
 
     Set<JobInfo> getBuildChildrenJobs();
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java

@@ -216,6 +216,8 @@ public interface IRunProcessService extends IService {
 
     ITalendProcessJavaProject getTalendJobJavaProject(Property property);
 
+    IFolder getCodeSrcFolder(ERepositoryObjectType type, String projectTechName);
+
     ITalendProcessJavaProject getTempJavaProject();
 
     void clearProjectRelatedSettings();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/ProjectManager.java

@@ -200,6 +200,10 @@ public final class ProjectManager {
      * return all the referenced projects of current project.
      */
     public List<Project> getAllReferencedProjects(boolean force) {
+        return getAllReferencedProjects(getCurrentProject(), force);
+    }
+
+    public List<Project> getAllReferencedProjects(Project targetProject, boolean force) {
         List<Project> allReferencedprojects = new ArrayList<Project>();
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IProxyRepositoryService.class)) {
             if (this.getCurrentProject() == null) {
@@ -212,7 +216,7 @@ public final class ProjectManager {
             IProxyRepositoryFactory factory = service.getProxyRepositoryFactory();
             if (factory != null) {
                 List<org.talend.core.model.properties.Project> rProjects = factory
-                        .getReferencedProjects(this.getCurrentProject());
+                        .getReferencedProjects(targetProject);
                 if (rProjects != null) {
                     for (org.talend.core.model.properties.Project p : rProjects) {
                         Project project = new Project(p);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/model/RepositoryConstants.java

@@ -98,6 +98,8 @@ public class RepositoryConstants {
 
     public static final String REPOSITORY_CLOUD_APAC_ID = "cloud_apac"; //$NON-NLS-1$
 
+    public static final String REPOSITORY_CLOUD_US_WEST_ID = "cloud_us_west"; //$NON-NLS-1$
+
     public static final String REPOSITORY_CLOUD_CUSTOM_ID = "cloud_custom"; //$NON-NLS-1$
 
     public static final String REPOSITORY_URL = "url"; //$NON-NLS-1$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/ui/login/connections/ConnectionUserPerReader.java

@@ -35,6 +35,7 @@ import org.talend.core.model.general.ConnectionBean;
 import org.talend.utils.json.JSONArray;
 import org.talend.utils.json.JSONException;
 import org.talend.utils.json.JSONObject;
+import org.talend.utils.security.EncryptedProperties;
 
 /**
  * DOC hwang class global comment. Detailled comment
@@ -126,8 +127,8 @@ public class ConnectionUserPerReader {
         if (!isHaveUserPer()) {
             createPropertyFile();
         }
-        try {
-            proper.load(new FileInputStream(perfile));
+        try (FileInputStream fi = new FileInputStream(perfile)) {
+            proper.load(fi);
             isRead = true;
         } catch (FileNotFoundException e) {
             e.printStackTrace();
@@ -176,9 +177,7 @@ public class ConnectionUserPerReader {
             }
             proper.setProperty("connection.define", usersJsonArray.toString());//$NON-NLS-1$
         }
-        try {
-
-            FileOutputStream out = new FileOutputStream(perfile);
+        try (FileOutputStream out = new FileOutputStream(perfile)) {
             proper.store(out, null);
         } catch (Exception e) {
             e.printStackTrace();
@@ -202,9 +201,8 @@ public class ConnectionUserPerReader {
                 proper.remove("connection.lastConnection"); //$NON-NLS-1$
             }
         }
-        FileOutputStream out;
-        try {
-            out = new FileOutputStream(perfile);
+        ;
+        try (FileOutputStream out = new FileOutputStream(perfile)) {
             proper.store(out, null);
         } catch (Exception e) {
             e.printStackTrace();
@@ -286,9 +284,7 @@ public class ConnectionUserPerReader {
         IPreferenceStore prefStore = PlatformUI.getPreferenceStore();
         proper.setProperty("connection.readRegistration", Integer.toString(prefStore.getInt("REGISTRATION_TRIES")));
         proper.setProperty("connection.readRegistrationDone", Integer.toString(prefStore.getInt("REGISTRATION_DONE")));
-        try {
-
-            FileOutputStream out = new FileOutputStream(perfile);
+        try (FileOutputStream out = new FileOutputStream(perfile)) {
             proper.store(out, null);
         } catch (Exception e) {
             e.printStackTrace();
@@ -317,8 +313,7 @@ public class ConnectionUserPerReader {
             String val = entry.getValue();
             proper.setProperty(key, val);
         }
-        try {
-            FileOutputStream out = new FileOutputStream(perfile);
+        try (FileOutputStream out = new FileOutputStream(perfile)) {
             proper.store(out, null);
         } catch (Exception e) {
             e.printStackTrace();
@@ -334,9 +329,7 @@ public class ConnectionUserPerReader {
         }
         IPreferenceStore prefStore = PlatformUI.getPreferenceStore();
         proper.setProperty("connection.licenseManagement", Integer.toString(prefStore.getInt("LICENSE_VALIDATION_DONE")));
-        try {
-
-            FileOutputStream out = new FileOutputStream(perfile);
+        try (FileOutputStream out = new FileOutputStream(perfile)) {
             proper.store(out, null);
         } catch (Exception e) {
             e.printStackTrace();
@@ -362,8 +355,7 @@ public class ConnectionUserPerReader {
             this.readProperties();
         }
         proper.setProperty("connection.installDone", Boolean.TRUE.toString()); //$NON-NLS-1$
-        try {
-            FileOutputStream out = new FileOutputStream(perfile);
+        try (FileOutputStream out = new FileOutputStream(perfile)) {
             proper.store(out, null);
         } catch (Exception e) {
             e.printStackTrace();

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/context/ContextNebulaGridComposite.java

@@ -472,7 +472,7 @@ public class ContextNebulaGridComposite extends AbstractContextTabEditComposite
             paramNameFound = true;
             paramName = NEW_PARAM_NAME + numParam;
             for (int i = 0; i < listParams.size(); i++) {
-                if (paramName.equals(listParams.get(i).getName())) {
+                if (paramName.equalsIgnoreCase(listParams.get(i).getName())) {
                     paramNameFound = false;
                 }
             }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/context/SelectRepositoryContextDialog.java

@@ -715,6 +715,13 @@ public class SelectRepositoryContextDialog extends SelectionDialog {
                 if (param != null && param.isBuiltIn()) {
                     return false;
                 }
+                if (param == null) {
+                    boolean containsSameParameterIgnoreCase = manager.getDefaultContext()
+                            .containsSameParameterIgnoreCase(paramType.getName());
+                    if (containsSameParameterIgnoreCase) {
+                        return false;
+                    }
+                }
             }
             return true;
         }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/token/DefaultTokenCollector.java

@@ -18,8 +18,8 @@ import org.talend.core.GlobalServiceRegister;
 import org.talend.core.prefs.ITalendCorePrefConstants;
 import org.talend.core.ui.CoreUIPlugin;
 import org.talend.core.ui.branding.IBrandingService;
-import org.talend.daikon.security.CryptoHelper;
 import org.talend.daikon.token.TokenGenerator;
+import org.talend.utils.security.StudioEncryption;
 
 import us.monoid.json.JSONObject;
 
@@ -47,7 +47,7 @@ public class DefaultTokenCollector extends AbstractTokenCollector {
     }
 
     public static String calcUniqueId() {
-        return TokenGenerator.generateMachineToken(new CryptoHelper(CryptoHelper.PASSPHRASE));
+        return TokenGenerator.generateMachineToken((src) -> StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM).encrypt(src));
     }
 
     /*

main/plugins/org.talend.core/src/main/java/org/talend/core/CoreService.java

@@ -54,6 +54,7 @@ import org.talend.core.model.metadata.QueryUtil;
 import org.talend.core.model.metadata.builder.ConvertionHelper;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.process.ElementParameterParser;
+import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.model.repository.IRepositoryViewObject;
@@ -280,9 +281,10 @@ public class CoreService implements ICoreService {
             ICodeGeneratorService codeGenService = (ICodeGeneratorService) GlobalServiceRegister.getDefault().getService(
                     ICodeGeneratorService.class);
             codeGenService.createRoutineSynchronizer().syncAllRoutinesForLogOn();
-            codeGenService.createRoutineSynchronizer().syncAllPigudfForLogOn();
+            if (ProcessUtils.isRequiredPigUDFs(null)) {
+                codeGenService.createRoutineSynchronizer().syncAllPigudfForLogOn();
+            }
         }
-
     }
 
     @Override

main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java

@@ -756,7 +756,7 @@ public class ProcessorUtilities {
 
     public static boolean hasMetadataDynamic(IProcess currentProcess, JobInfo jobInfo) {
         boolean hasDynamicMetadata = false;
-        if (GlobalServiceRegister.getDefault().isServiceRegistered(IDesignerCoreService.class)) {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IDesignerCoreService.class) && !isExportConfig()) {
             IDesignerCoreService designerCoreService =
                     (IDesignerCoreService) GlobalServiceRegister.getDefault().getService(IDesignerCoreService.class);
             for (INode node : currentProcess.getGraphicalNodes()) {
@@ -1231,14 +1231,23 @@ public class ProcessorUtilities {
     public static void cleanSourceFolder(IProgressMonitor progressMonitor, IProcess currentProcess,
             IProcessor processor) {
         try {
+            ITalendProcessJavaProject jobProject = processor.getTalendJavaProject();
+            // clean up source code
             IPath codePath = processor.getSrcCodePath().removeLastSegments(2);
-            IFolder srcFolder = processor.getTalendJavaProject().getProject().getFolder(codePath);
+            IFolder srcFolder = jobProject.getProject().getFolder(codePath);
             String jobPackageFolder = JavaResourcesHelper.getJobClassPackageFolder(currentProcess);
             for (IResource resource : srcFolder.members()) {
-                if (resource.getProjectRelativePath().toPortableString().endsWith(jobPackageFolder)) {
-                    break;
+                if (!resource.getProjectRelativePath().toPortableString().endsWith(jobPackageFolder)) {
+                    resource.delete(true, progressMonitor);
+                }
+            }
+            // clean up resources folder if needed
+            if (ProcessorUtilities.isExportConfig() && !designerCoreService.isNeedContextInJar(currentProcess)) {
+                for (IResource resource : jobProject.getResourcesFolder().members()) {
+                    if (resource.exists()) {
+                        resource.delete(true, progressMonitor);
+                    }
                 }
-                resource.delete(true, progressMonitor);
             }
         } catch (CoreException e) {
             ExceptionHandler.process(e);
@@ -2028,7 +2037,15 @@ public class ProcessorUtilities {
     public static String[] getCommandLine(String targetPlatform, boolean skipClasspathJar, boolean externalUse,
             String processId, String contextName, int statisticPort, int tracePort, String... codeOptions)
             throws ProcessorException {
+        return getCommandLine(targetPlatform, skipClasspathJar, externalUse,
+                processId, contextName, statisticPort, tracePort, false, codeOptions);
+    }
 
+    //ignoreCustomJVMSetting mean will generate the java command without the job VM setting in studio "Run job" setting and studio preferences "Run/Debug" setting
+    //only use for TDI-42443 in tRunJob
+    public static String[] getCommandLine(String targetPlatform, boolean skipClasspathJar, boolean externalUse,
+            String processId, String contextName, int statisticPort, int tracePort, boolean ignoreCustomJVMSetting, String... codeOptions)
+            throws ProcessorException {
         IProcessor processor = findProcessorFromJobList(processId, contextName, null);
         if (processor != null && targetPlatform.equals(processor.getTargetPlatform())) {
             if (processor.isProcessUnloaded()) {
@@ -2037,7 +2054,7 @@ public class ProcessorUtilities {
             boolean oldSkipClasspathJar = processor.isSkipClasspathJar();
             processor.setSkipClasspathJar(skipClasspathJar);
             try {
-                return processor.getCommandLine(true, externalUse, statisticPort, tracePort, codeOptions);
+                return processor.getCommandLine(true, externalUse, statisticPort, tracePort, ignoreCustomJVMSetting, codeOptions);
             } finally {
                 processor.setSkipClasspathJar(oldSkipClasspathJar);
             }
@@ -2055,7 +2072,7 @@ public class ProcessorUtilities {
         }
         // because all jobs are based one new way, set the flag "oldBuildJob" to false.
         return getCommandLine(false, skipClasspathJar, targetPlatform, externalUse, process,
-                selectedProcessItem.getProperty(), contextName, true, statisticPort, tracePort, codeOptions);
+                selectedProcessItem.getProperty(), contextName, true, statisticPort, tracePort, ignoreCustomJVMSetting, codeOptions);
     }
 
     /**
@@ -2129,6 +2146,14 @@ public class ProcessorUtilities {
     public static String[] getCommandLine(boolean oldBuildJob, boolean skipClasspathJar, String targetPlatform,
             boolean externalUse, IProcess currentProcess, Property property, String contextName, boolean needContext,
             int statisticPort, int tracePort, String... codeOptions) throws ProcessorException {
+        return getCommandLine(oldBuildJob, skipClasspathJar, targetPlatform,
+                externalUse, currentProcess, property, contextName, needContext,
+                statisticPort, tracePort, false, codeOptions);
+    }
+    
+    private static String[] getCommandLine(boolean oldBuildJob, boolean skipClasspathJar, String targetPlatform,
+            boolean externalUse, IProcess currentProcess, Property property, String contextName, boolean needContext,
+            int statisticPort, int tracePort, boolean ignoreCustomJVMSetting, String... codeOptions) throws ProcessorException {
         if (currentProcess == null) {
             return new String[] {};
         }
@@ -2141,7 +2166,7 @@ public class ProcessorUtilities {
         processor.setSkipClasspathJar(skipClasspathJar);
         processor.setTargetPlatform(targetPlatform);
         processor.setOldBuildJob(oldBuildJob);
-        return processor.getCommandLine(needContext, externalUse, statisticPort, tracePort, codeOptions);
+        return processor.getCommandLine(needContext, externalUse, statisticPort, tracePort, ignoreCustomJVMSetting, codeOptions);
     }
 
     public static String[] getCommandLine(boolean oldBuildJob, String targetPlatform, boolean externalUse,
@@ -2591,20 +2616,32 @@ public class ProcessorUtilities {
     }
 
     public static boolean isEsbJob(IProcess process) {
+        return isEsbJob(process, false);
+    }
+
+    public static boolean isEsbJob(IProcess process, boolean checkCurrentProcess) {
 
         if (process instanceof IProcess2) {
-            Set<JobInfo> infos = ProcessorUtilities.getChildrenJobInfo(((IProcess2) process).getProperty().getItem(), false);
 
-            for (JobInfo jobInfo : infos) {
-                ProcessType processType = jobInfo.getProcessItem().getProcess();
-                EList<NodeType> nodes = processType.getNode();
-                for (NodeType nodeType : nodes) {
-                    if (isEsbComponentName(nodeType.getComponentName())) {
+            if (checkCurrentProcess) {
+                for (INode n : process.getGraphicalNodes()) {
+                    if (isEsbComponentName(n.getComponent().getName())) {
                         return true;
                     }
                 }
-            }
+            } else {
+                Set<JobInfo> infos = ProcessorUtilities.getChildrenJobInfo(((IProcess2) process).getProperty().getItem(), false);
 
+                for (JobInfo jobInfo : infos) {
+                    ProcessType processType = jobInfo.getProcessItem().getProcess();
+                    EList<NodeType> nodes = processType.getNode();
+                    for (NodeType nodeType : nodes) {
+                        if (isEsbComponentName(nodeType.getComponentName())) {
+                            return true;
+                        }
+                    }
+                }
+            }
             return false;
         }
 

main/plugins/org.talend.designer.maven.aether/.classpath

@@ -1,29 +1,33 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
-	<classpathentry exported="true" kind="lib" path="lib/commons-codec.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/httpclient.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/httpcore.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/jcl-over-slf4j.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-api.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-connector-basic.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-impl.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-spi.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-transport-classpath.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-transport-file.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-transport-http.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-transport-wagon.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-util.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/plexus-utils.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/slf4j-api.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/wagon-provider-api.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-aether-provider-3.2.1.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-model-3.2.1.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-model-builder-3.2.1.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/maven-repository-metadata-3.2.1.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/plexus-interpolation-1.19.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/plexus-utils-3.0.17.jar"/>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
-	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
-	<classpathentry kind="src" path="src/main/java"/>
-	<classpathentry kind="output" path="target/classes"/>
-</classpath>
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-transport-file.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-transport-http.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/wagon-file.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/jsoup.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/wagon-http-shared.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/wagon-http.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/commons-codec.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/httpclient.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/httpcore.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/jcl-over-slf4j.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-api.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-connector-basic.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-impl.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-spi.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-transport-classpath.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-transport-wagon.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/maven-resolver-util.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/plexus-utils.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/slf4j-api.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/wagon-provider-api.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/maven-aether-provider-3.2.1.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/maven-model-3.2.1.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/maven-model-builder-3.2.1.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/maven-repository-metadata-3.2.1.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/plexus-interpolation-1.19.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/plexus-utils-3.0.17.jar"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
+	<classpathentry kind="src" path="src/main/java"/>
+	<classpathentry kind="output" path="target/classes"/>
+</classpath>

main/plugins/org.talend.designer.maven.aether/META-INF/MANIFEST.MF

@@ -26,13 +26,17 @@ Bundle-ClassPath: .,
  lib/maven-resolver-impl.jar,
  lib/maven-resolver-spi.jar,
  lib/maven-resolver-transport-classpath.jar,
- lib/maven-resolver-transport-file.jar,
- lib/maven-resolver-transport-http.jar,
  lib/maven-resolver-transport-wagon.jar,
  lib/maven-resolver-util.jar,
  lib/plexus-utils.jar,
  lib/slf4j-api.jar,
- lib/wagon-provider-api.jar
+ lib/wagon-provider-api.jar,
+ lib/jsoup.jar,
+ lib/wagon-http-shared.jar,
+ lib/wagon-http.jar,
+ lib/wagon-file.jar,
+ lib/maven-resolver-transport-file.jar,
+ lib/maven-resolver-transport-http.jar
 Export-Package: org.talend.designer.maven.aether,
  org.talend.designer.maven.aether.comparator,
  org.talend.designer.maven.aether.node,

main/plugins/org.talend.designer.maven.aether/build.properties

@@ -2,4 +2,10 @@ source.. = src/main/java/
 output.. = target/classes/
 bin.includes = META-INF/,\
                .,\
-               lib/
+               lib/,\
+               lib/jsoup.jar,\
+               lib/wagon-http-shared.jar,\
+               lib/wagon-http.jar,\
+               lib/wagon-file.jar,\
+               lib/maven-resolver-transport-file.jar,\
+               lib/maven-resolver-transport-http.jar

main/plugins/org.talend.designer.maven.aether/pom.xml

@@ -11,6 +11,7 @@
   <packaging>eclipse-plugin</packaging>
   <properties>
     <maven.resolver.version>1.3.1</maven.resolver.version>
+    <wagon.version>3.0.0</wagon.version>
   </properties>
 
   <dependencies>
@@ -45,10 +46,20 @@
       <version>${maven.resolver.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.apache.maven.resolver</groupId>
+      <groupId>org.apache.maven.wagon</groupId>
+      <artifactId>wagon-file</artifactId>
+      <version>${wagon.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.maven.wagon</groupId>
+      <artifactId>wagon-http</artifactId>
+      <version>${wagon.version}</version>
+    </dependency>
+    <dependency>
+    <groupId>org.apache.maven.resolver</groupId>
       <artifactId>maven-resolver-transport-file</artifactId>
       <version>${maven.resolver.version}</version>
-    </dependency>
+      </dependency>
     <dependency>
       <groupId>org.apache.maven.resolver</groupId>
       <artifactId>maven-resolver-transport-http</artifactId>

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/RepositorySystemFactory.java

@@ -17,24 +17,20 @@ import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
+import org.codehaus.plexus.PlexusContainerException;
 import org.eclipse.aether.DefaultRepositorySystemSession;
 import org.eclipse.aether.RepositorySystem;
 import org.eclipse.aether.artifact.Artifact;
 import org.eclipse.aether.artifact.DefaultArtifact;
-import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
 import org.eclipse.aether.deployment.DeployRequest;
-import org.eclipse.aether.impl.DefaultServiceLocator;
 import org.eclipse.aether.repository.Authentication;
 import org.eclipse.aether.repository.LocalRepository;
 import org.eclipse.aether.repository.RemoteRepository;
-import org.eclipse.aether.spi.connector.RepositoryConnectorFactory;
-import org.eclipse.aether.spi.connector.transport.TransporterFactory;
-import org.eclipse.aether.transport.file.FileTransporterFactory;
-import org.eclipse.aether.transport.http.HttpTransporterFactory;
 import org.eclipse.aether.util.artifact.SubArtifact;
 import org.eclipse.aether.util.listener.ChainedRepositoryListener;
 import org.eclipse.aether.util.listener.ChainedTransferListener;
 import org.eclipse.aether.util.repository.AuthenticationBuilder;
+import org.talend.designer.maven.aether.util.MavenLibraryResolverProvider;
 
 /**
  * created by wchen on Aug 10, 2017 Detailled comment
@@ -44,32 +40,16 @@ public class RepositorySystemFactory {
 
     private static Map<LocalRepository, DefaultRepositorySystemSession> sessions = new HashMap<LocalRepository, DefaultRepositorySystemSession>();
 
-    private static RepositorySystem system;
-
-    private static RepositorySystem newRepositorySystem() {
-        DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator();
-        locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class);
-        locator.addService(TransporterFactory.class, FileTransporterFactory.class);
-        locator.addService(TransporterFactory.class, HttpTransporterFactory.class);
-
-        locator.setErrorHandler(new DefaultServiceLocator.ErrorHandler() {
-
-            @Override
-            public void serviceCreationFailed(Class<?> type, Class<?> impl, Throwable exception) {
-                exception.printStackTrace();
-            }
-        });
-
-        return locator.getService(RepositorySystem.class);
-    }
-
-    private static DefaultRepositorySystemSession newRepositorySystemSession(String localRepositoryPath) {
+    private static DefaultRepositorySystemSession newRepositorySystemSession(String localRepositoryPath)
+            throws PlexusContainerException {
         LocalRepository localRepo = new LocalRepository(localRepositoryPath);
         DefaultRepositorySystemSession repositorySystemSession = sessions.get(localRepo);
         if (repositorySystemSession == null) {
             repositorySystemSession = MavenRepositorySystemUtils.newSession();
             repositorySystemSession
-                    .setLocalRepositoryManager(system.newLocalRepositoryManager(repositorySystemSession, localRepo));
+                    .setLocalRepositoryManager(
+                            MavenLibraryResolverProvider.newRepositorySystem().newLocalRepositoryManager(repositorySystemSession,
+                                    localRepo));
             repositorySystemSession.setTransferListener(new ChainedTransferListener());
             repositorySystemSession.setRepositoryListener(new ChainedRepositoryListener());
         }
@@ -81,9 +61,7 @@ public class RepositorySystemFactory {
             String userName, String password, String groupId, String artifactId, String classifier, String extension,
             String version) throws Exception {
         DefaultRepositorySystemSession session = null;
-        if (system == null) {
-            system = newRepositorySystem();
-        }
+        RepositorySystem system = MavenLibraryResolverProvider.newRepositorySystem();
         session = newRepositorySystemSession(localRepository);
 
         DeployRequest deployRequest = new DeployRequest();

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/DynamicDistributionAetherUtils.java

@@ -36,10 +36,11 @@ import org.eclipse.aether.artifact.Artifact;
 import org.eclipse.aether.artifact.DefaultArtifact;
 import org.eclipse.aether.collection.CollectRequest;
 import org.eclipse.aether.collection.DependencySelector;
-import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
 import org.eclipse.aether.graph.DefaultDependencyNode;
 import org.eclipse.aether.graph.Exclusion;
 import org.eclipse.aether.impl.DefaultServiceLocator;
+import org.eclipse.aether.internal.transport.wagon.PlexusWagonConfigurator;
+import org.eclipse.aether.internal.transport.wagon.PlexusWagonProvider;
 import org.eclipse.aether.repository.Authentication;
 import org.eclipse.aether.repository.LocalRepository;
 import org.eclipse.aether.repository.RemoteRepository;
@@ -48,10 +49,6 @@ import org.eclipse.aether.resolution.ArtifactRequest;
 import org.eclipse.aether.resolution.ArtifactResult;
 import org.eclipse.aether.resolution.VersionRangeRequest;
 import org.eclipse.aether.resolution.VersionRangeResult;
-import org.eclipse.aether.spi.connector.RepositoryConnectorFactory;
-import org.eclipse.aether.spi.connector.transport.TransporterFactory;
-import org.eclipse.aether.transport.file.FileTransporterFactory;
-import org.eclipse.aether.transport.http.HttpTransporterFactory;
 import org.eclipse.aether.util.artifact.JavaScopes;
 import org.eclipse.aether.util.graph.selector.AndDependencySelector;
 import org.eclipse.aether.util.graph.selector.OptionalDependencySelector;
@@ -111,11 +108,11 @@ public class DynamicDistributionAetherUtils {
 
         RepositorySystem repoSystem = null;
         if (multiThread) {
-            repoSystem = newRepositorySystem();
+            repoSystem = MavenLibraryResolverProvider.newRepositorySystemForResolver();
         } else {
             repoSystem = repoSystemMap.get(key);
             if (repoSystem == null) {
-                repoSystem = newRepositorySystem();
+                repoSystem = MavenLibraryResolverProvider.newRepositorySystemForResolver();
                 repoSystemMap.put(key, repoSystem);
             }
         }
@@ -283,7 +280,7 @@ public class DynamicDistributionAetherUtils {
         if (monitor == null) {
             monitor = new DummyDynamicMonitor();
         }
-        RepositorySystem repSystem = newRepositorySystem();
+        RepositorySystem repSystem = MavenLibraryResolverProvider.newRepositorySystemForResolver();
         RepositorySystemSession repSysSession = newSession(repSystem, localPath, monitor);
         updateDependencySelector((DefaultRepositorySystemSession) repSysSession, monitor);
 
@@ -329,7 +326,7 @@ public class DynamicDistributionAetherUtils {
         if (monitor == null) {
             monitor = new DummyDynamicMonitor();
         }
-        RepositorySystem repSystem = newRepositorySystem();
+        RepositorySystem repSystem = MavenLibraryResolverProvider.newRepositorySystemForResolver();
         RepositorySystemSession repSysSession = newSession(repSystem, localPath, monitor);
         updateDependencySelector((DefaultRepositorySystemSession) repSysSession, monitor);
 
@@ -411,15 +408,6 @@ public class DynamicDistributionAetherUtils {
     // }
     // }
 
-    private static RepositorySystem newRepositorySystem() {
-        DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator();
-        locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class);
-        locator.addService(TransporterFactory.class, FileTransporterFactory.class);
-        locator.addService(TransporterFactory.class, HttpTransporterFactory.class);
-
-        return locator.getService(RepositorySystem.class);
-    }
-
     private static RepositorySystemSession newSession(RepositorySystem system, String repositoryPath, IDynamicMonitor monitor)
             throws CoreException {
         DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession();

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/MavenLibraryResolverProvider.java

@@ -18,6 +18,12 @@ import java.util.Map;
 import org.apache.maven.model.License;
 import org.apache.maven.model.Model;
 import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
+import org.apache.maven.wagon.Wagon;
+import org.apache.maven.wagon.providers.file.FileWagon;
+import org.apache.maven.wagon.providers.http.HttpWagon;
+import org.codehaus.plexus.DefaultPlexusContainer;
+import org.codehaus.plexus.PlexusContainer;
+import org.codehaus.plexus.PlexusContainerException;
 import org.eclipse.aether.DefaultRepositorySystemSession;
 import org.eclipse.aether.RepositorySystem;
 import org.eclipse.aether.RepositorySystemSession;
@@ -25,6 +31,8 @@ import org.eclipse.aether.artifact.Artifact;
 import org.eclipse.aether.artifact.DefaultArtifact;
 import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
 import org.eclipse.aether.impl.DefaultServiceLocator;
+import org.eclipse.aether.internal.transport.wagon.PlexusWagonConfigurator;
+import org.eclipse.aether.internal.transport.wagon.PlexusWagonProvider;
 import org.eclipse.aether.repository.Authentication;
 import org.eclipse.aether.repository.LocalRepository;
 import org.eclipse.aether.repository.RemoteRepository;
@@ -34,6 +42,7 @@ import org.eclipse.aether.spi.connector.RepositoryConnectorFactory;
 import org.eclipse.aether.spi.connector.transport.TransporterFactory;
 import org.eclipse.aether.transport.file.FileTransporterFactory;
 import org.eclipse.aether.transport.http.HttpTransporterFactory;
+import org.eclipse.aether.transport.wagon.WagonTransporterFactory;
 import org.eclipse.aether.util.repository.AuthenticationBuilder;
 import org.eclipse.m2e.core.MavenPlugin;
 import org.talend.core.nexus.ArtifactRepositoryBean;
@@ -70,8 +79,8 @@ public class MavenLibraryResolverProvider {
         return instance;
     }
 
-    private MavenLibraryResolverProvider() {
-        defaultRepoSystem = newRepositorySystem();
+    private MavenLibraryResolverProvider() throws PlexusContainerException {
+        defaultRepoSystem = newRepositorySystemForResolver();
         defaultRepoSystemSession = newSession(defaultRepoSystem, getLocalMVNRepository());
         ArtifactRepositoryBean talendServer = TalendLibsServerManager.getInstance().getTalentArtifactServer();
         if (talendServer.getUserName() == null && talendServer.getPassword() == null) {
@@ -146,12 +155,44 @@ public class MavenLibraryResolverProvider {
         return repository;
     }
 
-    private RepositorySystem newRepositorySystem() {
+    public static RepositorySystem newRepositorySystem() throws PlexusContainerException {
+        DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator();
+        locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class);
+        // TUP-24695 change to wagon transporters
+        locator.addService(TransporterFactory.class, WagonTransporterFactory.class);
+
+        PlexusContainer pc = new DefaultPlexusContainer();
+
+        pc.addComponent(new HttpWagon(), Wagon.class, "http");
+        pc.addComponent(new FileWagon(), Wagon.class, "file");
+
+        WagonTransporterFactory tf = (WagonTransporterFactory) locator.getService(TransporterFactory.class);
+        tf.setWagonConfigurator(new PlexusWagonConfigurator(pc));
+        tf.setWagonProvider(new PlexusWagonProvider(pc));
+
+        locator.setErrorHandler(new DefaultServiceLocator.ErrorHandler() {
+
+            @Override
+            public void serviceCreationFailed(Class<?> type, Class<?> impl, Throwable exception) {
+                exception.printStackTrace();
+            }
+        });
+        return locator.getService(RepositorySystem.class);
+    }
+
+    public static RepositorySystem newRepositorySystemForResolver() {
         DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator();
         locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class);
         locator.addService(TransporterFactory.class, FileTransporterFactory.class);
         locator.addService(TransporterFactory.class, HttpTransporterFactory.class);
 
+        locator.setErrorHandler(new DefaultServiceLocator.ErrorHandler() {
+
+            @Override
+            public void serviceCreationFailed(Class<?> type, Class<?> impl, Throwable exception) {
+                exception.printStackTrace();
+            }
+        });
         return locator.getService(RepositorySystem.class);
     }
 

main/plugins/org.talend.designer.maven.tos/pom.xml

@@ -46,12 +46,6 @@
 			<version>${project.version}</version>
 			<type>pom</type>
 		</dependency>
-		<dependency>
-			<groupId>org.talend.studio</groupId>
-			<artifactId>features-maven-plugin-2-2-9-tos</artifactId>
-			<version>${project.version}</version>
-			<type>pom</type>
-		</dependency>		
 		<dependency>
 			<groupId>org.talend.studio</groupId>
 			<artifactId>maven-bundle-plugin-2-3-7-tos</artifactId>
@@ -76,18 +70,30 @@
 			<version>${project.version}</version>
 			<type>pom</type>
 		</dependency>
-        <dependency>
-            <groupId>org.talend.studio</groupId>
-            <artifactId>studio-maven-repository-zip</artifactId>
-            <version>${project.version}</version>
-            <type>pom</type>
-        </dependency>
         <dependency>        
             <groupId>org.talend.studio</groupId>
             <artifactId>build-helper-maven-plugin-3-0-0-tos</artifactId>
             <version>${project.version}</version>
             <type>pom</type>
         </dependency>
+		<dependency>
+			<groupId>org.talend.studio</groupId>
+			<artifactId>karaf-maven-plugin-4-2-4-tos</artifactId>
+			<version>${project.version}</version>
+			<type>pom</type>
+		</dependency>
+        <dependency>
+            <groupId>org.talend.studio</groupId>
+            <artifactId>studio-maven-repository-zip</artifactId>
+            <version>${project.version}</version>
+            <type>pom</type>
+        </dependency>
+		<dependency>
+			<groupId>org.talend.studio</groupId>
+			<artifactId>karaf-maven-plugin-4-2-4-tos</artifactId>
+			<version>${project.version}</version>
+			<type>pom</type>
+		</dependency>		
 	</dependencies>
 	<profiles>
 		<profile>

main/plugins/org.talend.designer.maven.tos/resources/build/plugins/karaf-maven-plugin-4-2-4/pom.xml

@@ -7,41 +7,30 @@
 		<version>7.3.1-SNAPSHOT</version>
 		<relativePath>../../../</relativePath>
 	</parent>
-	<artifactId>features-maven-plugin-2-2-9-tos</artifactId>
+	<artifactId>karaf-maven-plugin-4-2-4-tos</artifactId>
 	<packaging>pom</packaging>
 	<dependencies>
-	
 		<dependency>
 			<groupId>org.apache.karaf.tooling</groupId>
-			<artifactId>features-maven-plugin</artifactId>
-			<version>2.2.9</version>
+			<artifactId>karaf-maven-plugin</artifactId>
+			<version>4.2.4</version>
 		</dependency>
-		
 		<dependency>
-			<groupId>org.apache.felix</groupId>
-			<artifactId>org.apache.felix.bundlerepository</artifactId>
-			<version>1.6.4</version>
+			<groupId>org.slf4j</groupId>
+			<artifactId>jcl-over-slf4j</artifactId>
+			<version>1.7.12</version>
 		</dependency>
-		
 		<dependency>
-			<groupId>org.osgi</groupId>
-			<artifactId>org.osgi.core</artifactId>
-			<version>4.2.0</version>
+			<groupId>org.apache.maven.shared</groupId>
+			<artifactId>maven-dependency-tree</artifactId>
+			<version>3.0.1</version>
 		</dependency>
-		
 		<dependency>
-			<groupId>org.easymock</groupId>
-			<artifactId>easymock</artifactId>
-			<version>3.1</version>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-framework-bom</artifactId>
+			<version>5.0.2.RELEASE</version>
+			<type>pom</type>
 		</dependency>
-		
-                <!-- Required by commons-lang-2.6.pom -->
-                <dependency>
-                    <groupId>org.apache.commons</groupId>
-                    <artifactId>commons-parent</artifactId>
-                    <version>17</version>
-                    <type>pom</type>
-                </dependency>		
 	</dependencies>
 	<build>
 		<plugins>

main/plugins/org.talend.designer.maven.tos/resources/build/pom.xml

@@ -10,9 +10,9 @@
 	<artifactId>studio-maven-repository-build</artifactId>
 	<packaging>pom</packaging>
 	<modules>
-		<module>plugins/features-maven-plugin-2-2-9</module>
 		<module>plugins/maven-bundle-plugin-2-3-7</module>
 		<module>plugins/maven-bundle-plugin-2-5-3</module>
+		<module>plugins/karaf-maven-plugin-4-2-4</module>
 		<module>plugins/maven-install-plugin-2-5-1</module>
 		<module>plugins/talend-compiler-plugin</module>
 		<module>plugins/build-helper-maven-plugin-3-0-0</module>

main/plugins/org.talend.designer.maven.tos/resources/tcompv1/pom.xml

@@ -11,7 +11,7 @@
     <packaging>pom</packaging>
     
     <properties>
-        <tcomp.version>1.1.10</tcomp.version>
+        <tcomp.version>1.1.14</tcomp.version>
         <slf4j.version>1.7.25</slf4j.version>
     </properties>
     

main/plugins/org.talend.designer.maven.ui/src/main/java/messages.properties

@@ -11,4 +11,5 @@ ProjectPomProjectSettingPage_FilterPomLabel=Filter to use to generate poms:
 ProjectPomProjectSettingPage_FilterErrorMessage=Invalid filter: {0}
 ProjectPomProjectSettingPage.syncAllPomsButtonText=Force full re-synchronize poms
 AbstractPersistentProjectSettingPage.syncAllPoms=Do you want to update all poms? \n This operation might take long time depends on your project size.
-MavenProjectSettingPage.filterExampleMessage=Filter examples:\nlabel=myJob       \t\t\t\t=> Generate only the job named "myJob"\n!(label=myJob)    \t\t\t\t=> Generate any job except the one named "myJob"\n(path=folder1/folder2)  \t\t\t=> Generate any job in the folder "folder1/folder2"\n(path=folder1/folder2)or(label=myJob)\t=> Generate any job in the folder "folder1/folder2" or named "myJob"\n(label=myJob)and(version=0.2)	\t=> Generate only the job named "myJob" with version 0.2\n!((label=myJob)and(version=0.1))	\t=> Generate every jobs except the "myJob" version 0.1
+MavenProjectSettingPage.filterExampleMessage=Filter examples:\nlabel=myJob       \t\t\t\t=> Generate only the job named "myJob"\n!(label=myJob)    \t\t\t\t=> Generate any job except the one named "myJob"\n(path=folder1/folder2)  \t\t\t=> Generate any job in the folder "folder1/folder2"\n(path=folder1/folder2)or(label=myJob)\t=> Generate any job in the folder "folder1/folder2" or named "myJob"\n(label=myJob)and(version=0.2)	\t=> Generate only the job named "myJob" with version 0.2\n!((label=myJob)and(version=0.1))	\t=> Generate every jobs except the "myJob" version 0.1
+MavenProjectSettingPage.refModuleText=Set reference project modules in profile

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/setting/preference/M2eUserSettingForTalendLoginTask.java

@@ -294,20 +294,25 @@ public class M2eUserSettingForTalendLoginTask extends AbstractLoginTask {
         boolean isLocal = isLocalRepository();
         IPath localRepoPath = null;
         if (!isLocal) {
-            String mvnHome = System.getenv("M2_HOME"); //$NON-NLS-1$
-            if (mvnHome == null) {
-                mvnHome = System.getenv("MAVEN_HOME"); //$NON-NLS-1$
-            }
-            if (StringUtils.isNotBlank(mvnHome)) {
-                File globalSettings = new File(mvnHome).toPath().resolve("conf").resolve("settings.xml").toFile(); //$NON-NLS-1$ //$NON-NLS-2$
-                if (globalSettings.exists()) {
-                    DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
-                    Document document = builder.parse(globalSettings);
-                    Node node = document.getElementsByTagName("localRepository").item(0); //$NON-NLS-1$
-                    if (node != null) {
-                        String repoPath = node.getTextContent();
-                        if (StringUtils.isNotBlank(repoPath)) {
-                            localRepoPath = new Path(repoPath);
+            String customMavenRepoistory = System.getProperty("maven.local.repository");
+            if (customMavenRepoistory != null) {
+                localRepoPath = new Path(customMavenRepoistory);
+            } else {
+                String mvnHome = System.getenv("M2_HOME"); //$NON-NLS-1$
+                if (mvnHome == null) {
+                    mvnHome = System.getenv("MAVEN_HOME"); //$NON-NLS-1$
+                }
+                if (StringUtils.isNotBlank(mvnHome)) {
+                    File globalSettings = new File(mvnHome).toPath().resolve("conf").resolve("settings.xml").toFile(); //$NON-NLS-1$ //$NON-NLS-2$
+                    if (globalSettings.exists()) {
+                        DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
+                        Document document = builder.parse(globalSettings);
+                        Node node = document.getElementsByTagName("localRepository").item(0); //$NON-NLS-1$
+                        if (node != null) {
+                            String repoPath = node.getTextContent();
+                            if (StringUtils.isNotBlank(repoPath)) {
+                                localRepoPath = new Path(repoPath);
+                            }
                         }
                     }
                 }

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/setting/project/page/MavenProjectSettingPage.java

@@ -50,6 +50,8 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
 
 	private IPreferenceStore preferenceStore;
 
+    private Button useProfileModuleCheckbox;
+
 	public MavenProjectSettingPage() {
 		noDefaultAndApplyButton();
 	}
@@ -84,6 +86,10 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
         filterExampleLable.setText(Messages.getString("MavenProjectSettingPage.filterExampleMessage")); //$NON-NLS-1$
         filterExampleLable.setLayoutData(new GridData(SWT.FILL, SWT.CENTER, true, false));
 
+        useProfileModuleCheckbox = new Button(parent, SWT.CHECK);
+        useProfileModuleCheckbox.setText(Messages.getString("MavenProjectSettingPage.refModuleText")); //$NON-NLS-1$
+        useProfileModuleCheckbox.setSelection(preferenceStore.getBoolean(MavenConstants.USE_PROFILE_MODULE));
+
         filterText.setText(filter);
 		filterText.addModifyListener(new ModifyListener() {
 
@@ -115,6 +121,7 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
 			public void widgetSelected(SelectionEvent event) {
 				try {
                     preferenceStore.setValue(MavenConstants.POM_FILTER, getRealVersionFilter(filter));
+                    preferenceStore.setValue(MavenConstants.USE_PROFILE_MODULE, useProfileModuleCheckbox.getSelection());
 					new AggregatorPomsHelper().syncAllPoms();
 				} catch (Exception e) {
 					ExceptionHandler.process(e);
@@ -133,6 +140,7 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
 		boolean ok = super.performOk();
 		if (preferenceStore != null) {
             preferenceStore.setValue(MavenConstants.POM_FILTER, getRealVersionFilter(filter));
+            preferenceStore.setValue(MavenConstants.USE_PROFILE_MODULE, useProfileModuleCheckbox.getSelection());
 		}
 		return ok;
 	}

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/AggregatorPomsHelper.java

@@ -12,24 +12,23 @@
 // ============================================================================
 package org.talend.designer.maven.tools;
 
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_AGGREGATORS;
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_BEANS;
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_CODES;
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_JOBS;
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_PIGUDFS;
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_POMS;
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_ROUTINES;
+import static org.talend.designer.maven.model.TalendJavaProjectConstants.*;
 
 import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
-import java.util.ListIterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.maven.model.Activation;
 import org.apache.maven.model.Model;
+import org.apache.maven.model.Profile;
 import org.eclipse.core.resources.IContainer;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
@@ -56,7 +55,6 @@ import org.talend.core.context.Context;
 import org.talend.core.context.RepositoryContext;
 import org.talend.core.model.general.ILibrariesService;
 import org.talend.core.model.general.ModuleNeeded;
-import org.talend.core.model.general.Project;
 import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.ProjectReference;
@@ -77,6 +75,7 @@ import org.talend.core.runtime.services.IFilterService;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.designer.core.ICamelDesignerCoreService;
 import org.talend.designer.maven.launch.MavenPomCommandLauncher;
+import org.talend.designer.maven.model.MavenSystemFolders;
 import org.talend.designer.maven.model.TalendJavaProjectConstants;
 import org.talend.designer.maven.model.TalendMavenConstants;
 import org.talend.designer.maven.template.MavenTemplateManager;
@@ -110,27 +109,25 @@ public class AggregatorPomsHelper {
         return projectTechName;
     }
 
-    public void createRootPom(List<String> modules, boolean force, IProgressMonitor monitor) throws Exception {
+    public void createRootPom(Model model, boolean force, IProgressMonitor monitor)
+            throws Exception {
         IFile pomFile = getProjectRootPom();
         if (force || !pomFile.exists()) {
-            Map<String, Object> parameters = new HashMap<String, Object>();
-            parameters.put(MavenTemplateManager.KEY_PROJECT_NAME, projectTechName);
-            Model model = MavenTemplateManager.getCodeProjectTemplateModel(parameters);
-            if (modules != null && !modules.isEmpty()) {
-                model.setModules(modules);
-            }
             PomUtil.savePom(monitor, model, pomFile);
         }
     }
 
     public void createRootPom(IProgressMonitor monitor) throws Exception {
+        Model newModel = getCodeProjectTemplateModel();
         IFile pomFile = getProjectRootPom();
-        List<String> modules = null;
         if (pomFile != null && pomFile.exists()) {
-            Model model = MavenPlugin.getMavenModelManager().readMavenModel(pomFile);
-            modules = model.getModules();
+            Model oldModel = MavenPlugin.getMavenModelManager().readMavenModel(pomFile);
+            List<Profile> profiles = oldModel.getProfiles().stream()
+                    .filter(profile -> matchModuleProfile(profile.getId(), projectTechName)).collect(Collectors.toList());
+            newModel.setModules(oldModel.getModules());
+            newModel.getProfiles().addAll(profiles);
         }
-        createRootPom(modules, true, monitor);
+        createRootPom(newModel, true, monitor);
     }
 
     public void installRootPom(boolean force) throws Exception {
@@ -290,37 +287,40 @@ public class AggregatorPomsHelper {
         }
     }
 
-    public void updateRefProjectModules(List<ProjectReference> references) {
+    public void updateRefProjectModules(List<ProjectReference> references, IProgressMonitor monitor) {
         if (!needUpdateRefProjectModules()) {
             return;
         }
         try {
-            List<String> modules = new ArrayList<>();
-            for (ProjectReference reference : references) {
-                String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
-                String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
-                modules.add(modulePath);
-            }
-
-            Project mainProject = ProjectManager.getInstance().getCurrentProject();
-            IFolder mainPomsFolder = new AggregatorPomsHelper(mainProject.getTechnicalLabel()).getProjectPomsFolder();
-            IFile mainPomFile = mainPomsFolder.getFile(TalendMavenConstants.POM_FILE_NAME);
-
-            Model model = MavenPlugin.getMavenModelManager().readMavenModel(mainPomFile);
-            List<String> oldModules = model.getModules();
-            if (oldModules == null) {
-                oldModules = new ArrayList<>();
-            }
-            ListIterator<String> iterator = oldModules.listIterator();
-            while (iterator.hasNext()) {
-                String modulePath = iterator.next();
-                if (modulePath.startsWith("../../")) { //$NON-NLS-1$
-                    iterator.remove();
+            Model model = MavenPlugin.getMavenModelManager().readMavenModel(getProjectRootPom());
+            if (PomIdsHelper.useProfileModule()) {
+                List<Profile> profiles = collectRefProjectProfiles(references);
+                Iterator<Profile> iterator = model.getProfiles().listIterator();
+                while (iterator.hasNext()) {
+                    Profile profile = iterator.next();
+                    if (matchModuleProfile(profile.getId(), projectTechName)) {
+                        iterator.remove();
+                    }
                 }
+                model.getProfiles().addAll(profiles);
+            } else {
+                List<String> refPrjectModules = new ArrayList<>();
+                references.forEach(reference -> {
+                    String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
+                    String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
+                    refPrjectModules.add(modulePath);
+                });
+                List<String> modules = model.getModules();
+                Iterator<String> iterator = modules.listIterator();
+                while (iterator.hasNext()) {
+                    String modulePath = iterator.next();
+                    if (modulePath.startsWith("../../")) { //$NON-NLS-1$
+                        iterator.remove();
+                    }
+                }
+                modules.addAll(refPrjectModules);
             }
-            oldModules.addAll(modules);
-
-            PomUtil.savePom(null, model, mainPomFile);
+            createRootPom(model, true, monitor);
         } catch (Exception e) {
             ExceptionHandler.process(e);
         }
@@ -494,6 +494,10 @@ public class AggregatorPomsHelper {
         return null;
     }
 
+    public IFolder getCodeSrcFolder(ERepositoryObjectType codeType) {
+        return getCodeFolder(codeType).getFolder(MavenSystemFolders.JAVA.getPath());
+    }
+
     public IFolder getProcessFolder(ERepositoryObjectType type) {
         return getProjectPomsFolder().getFolder(DIR_JOBS).getFolder(type.getFolder());
     }
@@ -710,47 +714,48 @@ public class AggregatorPomsHelper {
         return null;
     }
 
-    private void collectModules(List<String> modules) {
-        IRunProcessService service = getRunProcessService();
-        if (service != null) {
-            modules.add(
-                    getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.ROUTINES).getProjectPom()));
-            if (ProcessUtils.isRequiredPigUDFs(null)) {
-                modules.add(
-                        getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.PIG_UDF).getProjectPom()));
-            }
-            if (ProcessUtils.isRequiredBeans(null)) {
-                modules.add(getModulePath(service
-                        .getTalendCodeJavaProject(ERepositoryObjectType.valueOf("BEANS")) //$NON-NLS-1$
-                        .getProjectPom()));
-            }
+    private List<Profile> collectRefProjectProfiles(List<ProjectReference> references) throws CoreException {
+        if (!needUpdateRefProjectModules()) {
+            Model model = MavenPlugin.getMavenModelManager().readMavenModel(getProjectRootPom());
+            List<Profile> profiles = model.getProfiles();
+            return profiles.stream().filter(profile -> matchModuleProfile(profile.getId(), projectTechName))
+                    .collect(Collectors.toList());
         }
-        if (needUpdateRefProjectModules()) {
-            List<ProjectReference> references =
-                    ProjectManager.getInstance().getCurrentProject().getProjectReferenceList(true);
-            for (ProjectReference reference : references) {
-                String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
-                String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
-                modules.add(modulePath);
-            }
-        } else {
-            Project mainProject = ProjectManager.getInstance().getCurrentProject();
-            IFolder mainPomsFolder = new AggregatorPomsHelper(mainProject.getTechnicalLabel()).getProjectPomsFolder();
-            IFile mainPomFile = mainPomsFolder.getFile(TalendMavenConstants.POM_FILE_NAME);
-            try {
-                Model model = MavenPlugin.getMavenModelManager().readMavenModel(mainPomFile);
-                List<String> oldModules = model.getModules();
-                if (oldModules != null) {
-                    for (String modulePath : oldModules) {
-                        if (modulePath.startsWith("../../")) { //$NON-NLS-1$
-                            modules.add(modulePath);
-                        }
-                    }
-                }
-            } catch (CoreException e) {
-                ExceptionHandler.process(e);
-            }
+        if (references == null) {
+            references = ProjectManager.getInstance().getCurrentProject().getProjectReferenceList(true);
         }
+        List<Profile> profiles = new ArrayList<>();
+        references.forEach(reference -> {
+            String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
+            String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
+            Profile profile = new Profile();
+            profile.setId((projectTechName + "_" + refProjectTechName).toLowerCase()); //$NON-NLS-1$
+            Activation activation = new Activation();
+            activation.setActiveByDefault(true);
+            profile.setActivation(activation);
+            profile.getModules().add(modulePath);
+            profiles.add(profile);
+        });
+        return profiles;
+    }
+
+    private List<String> collectRefProjectModules(List<ProjectReference> references) throws CoreException {
+        if (!needUpdateRefProjectModules()) {
+            Model model = MavenPlugin.getMavenModelManager().readMavenModel(getProjectRootPom());
+            return model.getModules().stream().filter(modulePath -> modulePath.startsWith("../../")) //$NON-NLS-1$
+                    .collect(Collectors.toList());
+        }
+        if (references == null) {
+            references = ProjectManager.getInstance().getCurrentProject().getProjectReferenceList(true);
+        }
+        List<String> modules = new ArrayList<>();
+        references.forEach(reference -> {
+            String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
+            String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
+            modules.add(modulePath);
+        });
+        return modules;
+
     }
 
     public boolean needUpdateRefProjectModules() {
@@ -788,7 +793,13 @@ public class AggregatorPomsHelper {
         monitor.beginTask("", size); //$NON-NLS-1$
         // project pom
         monitor.subTask("Synchronize project pom"); //$NON-NLS-1$
-        createRootPom(null, true, monitor);
+        Model model = getCodeProjectTemplateModel();
+        if (PomIdsHelper.useProfileModule()) {
+            model.getProfiles().addAll(collectRefProjectProfiles(null));
+        } else {
+            model.getModules().addAll(collectRefProjectModules(null));
+        }
+        createRootPom(model, true, monitor);
         installRootPom(true);
         monitor.worked(1);
         if (monitor.isCanceled()) {
@@ -841,8 +852,9 @@ public class AggregatorPomsHelper {
         }
         // sync project pom again with all modules.
         monitor.subTask("Synchronize project pom with modules"); //$NON-NLS-1$
-        collectModules(modules);
-        createRootPom(modules, true, monitor);
+        collectCodeModules(modules);
+        model.getModules().addAll(modules);
+        createRootPom(model, true, monitor);
         installRootPom(true);
         monitor.worked(1);
         if (monitor.isCanceled()) {
@@ -851,6 +863,21 @@ public class AggregatorPomsHelper {
         monitor.done();
     }
 
+    private void collectCodeModules(List<String> modules) {
+        // collect codes modules
+        IRunProcessService service = getRunProcessService();
+        if (service != null) {
+            modules.add(getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.ROUTINES).getProjectPom()));
+            if (ProcessUtils.isRequiredPigUDFs(null)) {
+                modules.add(getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.PIG_UDF).getProjectPom()));
+            }
+            if (ProcessUtils.isRequiredBeans(null)) {
+                modules.add(getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.valueOf("BEANS")) //$NON-NLS-1$
+                        .getProjectPom()));
+            }
+        }
+    }
+
     /**
      * Check if is a esb data service job
      *
@@ -879,6 +906,18 @@ public class AggregatorPomsHelper {
         return null;
     }
 
+    private Model getCodeProjectTemplateModel() {
+        Map<String, Object> parameters = new HashMap<String, Object>();
+        parameters.put(MavenTemplateManager.KEY_PROJECT_NAME, projectTechName);
+        return MavenTemplateManager.getCodeProjectTemplateModel(parameters);
+    }
+
+    public static boolean matchModuleProfile(String profileId, String projectTechName) {
+        // FIXME get profile id from extension point.
+        List<String> otherProfiles = Arrays.asList("docker", "cloud-publisher", "nexus"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+        return !otherProfiles.contains(profileId) && StringUtils.startsWithIgnoreCase(profileId, projectTechName + "_");
+    }
+
     private static IRunProcessService getRunProcessService() {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
             IRunProcessService runProcessService =

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/ProcessorDependenciesManager.java

@@ -167,6 +167,7 @@ public class ProcessorDependenciesManager {
         if (modulesNeeded.isEmpty()) {
             modulesNeeded = processor.getNeededModules(TalendProcessOptionConstants.MODULES_WITH_JOBLET);
         }
+        processor.updateModulesAfterSetLog4j(modulesNeeded);
         neededLibraries.addAll(modulesNeeded);
 
         // add testcase modules

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/ProjectPomManager.java

@@ -69,13 +69,7 @@ public class ProjectPomManager {
         if (monitor == null) {
             monitor = new NullProgressMonitor();
         }
-        Model projectModel = MODEL_MANAGER.readMavenModel(projectPomFile);
-        Model templateModel = MavenTemplateManager.getCodeProjectTemplateModel();
-        for (String module : projectModel.getModules()) {
-            templateModel.addModule(module);
-        }
-
-        PomUtil.savePom(monitor, templateModel, projectPomFile);
+        new AggregatorPomsHelper().createRootPom(monitor);
     }
 
     /**

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/AbstractMavenProcessorPom.java

@@ -220,6 +220,29 @@ public abstract class AbstractMavenProcessorPom extends CreateMavenBundleTemplat
                     MavenArtifact mvnArtifact = MavenUrlHelper.parseMvnUrl(module.getMavenUri());
                     include.setValue(mvnArtifact.getGroupId() + ":" + mvnArtifact.getArtifactId()); //$NON-NLS-1$
                 }
+                
+                //removing digital signatures from uber jar
+                Xpp3Dom filters = new Xpp3Dom("filters"); //$NON-NLS-1$
+                Xpp3Dom filter = new Xpp3Dom("filter"); //$NON-NLS-1$
+                Xpp3Dom artifact = new Xpp3Dom("artifact"); //$NON-NLS-1$
+                artifact.setValue("*:*");
+                Xpp3Dom filterExcludes = new Xpp3Dom("excludes"); //$NON-NLS-1$
+                Xpp3Dom excludeSF = new Xpp3Dom("exclude");
+                excludeSF.setValue("META-INF/*.SF");
+                Xpp3Dom excludeDSA = new Xpp3Dom("exclude");
+                excludeDSA.setValue("META-INF/*.DSA");
+                Xpp3Dom excludeRSA = new Xpp3Dom("exclude");
+                excludeRSA.setValue("META-INF/*.RSA");
+
+                filterExcludes.addChild(excludeSF);
+                filterExcludes.addChild(excludeDSA);
+                filterExcludes.addChild(excludeRSA);
+                
+                filter.addChild(artifact);
+                filter.addChild(filterExcludes);
+                filters.addChild(filter);
+                configuration.addChild(filters);
+                
                 plugins.add(shade);
             }
         }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenStandardJobOSGiPom.java

@@ -152,11 +152,13 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
         model.addProperty("talend.job.finalName", "${talend.job.name}-bundle-${project.version}");
         Build build = model.getBuild();
         
-        IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IESBService.class)) {
+            IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
 
-        if (isServiceOperation || service.isRESTService((ProcessItem) getJobProcessor().getProperty().getItem())
-                || isRouteOperation(getJobProcessor().getProperty())) {
-            build.addPlugin(addSkipDockerMavenPlugin());
+            if (isServiceOperation || service.isRESTService((ProcessItem) getJobProcessor().getProperty().getItem())
+                    || isRouteOperation(getJobProcessor().getProperty())) {
+                build.addPlugin(addSkipDockerMavenPlugin());
+            }
         }
         
         if (isServiceOperation) {
@@ -283,25 +285,25 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
         List<IRepositoryViewObject> serviceRepoList = null;
 
         boolean isDataServiceOperation = false;
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IESBService.class)) {
+            IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
+            try {
+                IProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
+                serviceRepoList = factory.getAll(ERepositoryObjectType.valueOf(ERepositoryObjectType.class, "SERVICES"));
 
-        IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
-
-        try {
-            IProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
-            serviceRepoList = factory.getAll(ERepositoryObjectType.valueOf(ERepositoryObjectType.class, "SERVICES"));
-
-            for (IRepositoryViewObject serviceItem : serviceRepoList) {
-                if (service != null) {
-                    List<String> jobIds = service.getSerivceRelatedJobIds(serviceItem.getProperty().getItem());
-                    if (jobIds.contains(property.getId())) {
-                        isDataServiceOperation = true;
-                        break;
+                for (IRepositoryViewObject serviceItem : serviceRepoList) {
+                    if (service != null) {
+                        List<String> jobIds = service.getSerivceRelatedJobIds(serviceItem.getProperty().getItem());
+                        if (jobIds.contains(property.getId())) {
+                            isDataServiceOperation = true;
+                            break;
+                        }
                     }
                 }
-            }
 
-        } catch (PersistenceException e) {
-            ExceptionHandler.process(e);
+            } catch (PersistenceException e) {
+                ExceptionHandler.process(e);
+            }
         }
 
         return isDataServiceOperation;
@@ -311,32 +313,33 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
         List<IRepositoryViewObject> routeRepoList = null;
 
         boolean isRouteOperation = false;
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IESBService.class)) {
 
-        IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
+            IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
 
-        try {
-            IProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
-            routeRepoList = factory.getAll(ERepositoryObjectType.valueOf(ERepositoryObjectType.class, "ROUTE"));
+            try {
+                IProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
+                routeRepoList = factory.getAll(ERepositoryObjectType.valueOf(ERepositoryObjectType.class, "ROUTE"));
 
-            for (IRepositoryViewObject routeItem : routeRepoList) {
-                if (service != null) {
+                for (IRepositoryViewObject routeItem : routeRepoList) {
+                    if (service != null) {
 
-                    List<Relation> relations = RelationshipItemBuilder.getInstance().getItemsRelatedTo(routeItem.getId(),
-                            routeItem.getVersion(), RelationshipItemBuilder.JOB_RELATION);
-                    for (Relation relation : relations) {
-                        if (relation.getType() == RelationshipItemBuilder.JOB_RELATION) {
-                            if (relation.getId().equals(property.getId())) {
-                                isRouteOperation = true;
+                        List<Relation> relations = RelationshipItemBuilder.getInstance().getItemsRelatedTo(routeItem.getId(),
+                                routeItem.getVersion(), RelationshipItemBuilder.JOB_RELATION);
+                        for (Relation relation : relations) {
+                            if (relation.getType() == RelationshipItemBuilder.JOB_RELATION) {
+                                if (relation.getId().equals(property.getId())) {
+                                    isRouteOperation = true;
+                                }
                             }
                         }
                     }
                 }
+
+            } catch (PersistenceException e) {
+                ExceptionHandler.process(e);
             }
-
-        } catch (PersistenceException e) {
-            ExceptionHandler.process(e);
         }
-
         return isRouteOperation;
     }
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/PomIdsHelper.java

@@ -260,6 +260,12 @@ public class PomIdsHelper {
         return manager.getValue(MavenConstants.POM_FILTER);
     }
 
+    public static boolean useProfileModule() {
+        String projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
+        ProjectPreferenceManager manager = getPreferenceManager(projectTechName);
+        return manager.getBoolean(MavenConstants.USE_PROFILE_MODULE);
+    }
+
     private static String getGroupId(String projectTechName, String baseName, Property property) {
         if (projectTechName == null) {
             projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/PomUtil.java

@@ -547,7 +547,7 @@ public class PomUtil {
 
             File resolve = null;
             try {
-                resolve = TalendMavenResolver.getMavenResolver().resolve(localMavenUri);
+                resolve = TalendMavenResolver.resolve(localMavenUri);
             } catch (IOException | RuntimeException e) {
                 resolve = null;
             }

main/plugins/org.talend.libraries.jdbc.exasol/pom.xml

@@ -45,9 +45,9 @@
                     <configuration>
                         <artifactItems>
                         	<artifactItem>
-                                <groupId>org.talend.libraries</groupId>
+                                <groupId>org.talend.components.lib</groupId>
                                 <artifactId>talend-db-exasol</artifactId>
-                                <version>2.1.3</version>
+                                <version>2.1.4</version>
                                 <type>jar</type>
                                 <overWrite>true</overWrite>
                                 <outputDirectory>${libs.dir}</outputDirectory>

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/ui/dialogs/ConfigModuleDialog.java

@@ -116,6 +116,10 @@ public class ConfigModuleDialog extends TitleAreaDialog implements IConfigModule
     private String defaultURIValue = "";
 
     private Set<String> jarsAvailable;
+    
+    private boolean useCustom = false;
+    
+    private String customURI = null;
 
     /**
      * DOC wchen InstallModuleDialog constructor comment.
@@ -553,11 +557,13 @@ public class ConfigModuleDialog extends TitleAreaDialog implements IConfigModule
                 layoutWarningComposite(false, defaultUriTxt.getText());
                 if (useCustomBtn.getSelection()) {
                     customUriText.setEnabled(true);
+                    useCustom = true;
                     if ("".equals(customUriText.getText())) {
                         customUriText.setText(ModuleMavenURIUtils.MVNURI_TEMPLET);
                     }
                 } else {
                     customUriText.setEnabled(false);
+                    useCustom = false;
                 }
                 checkFieldsError();
             }
@@ -807,7 +813,6 @@ public class ConfigModuleDialog extends TitleAreaDialog implements IConfigModule
     @Override
     protected void okPressed() {
         String originalURI = null;
-        String customURI = null;
         originalURI = defaultUriTxt.getText().trim();
         defaultURI = originalURI;
         if (useCustomBtn.getSelection()) {
@@ -889,6 +894,9 @@ public class ConfigModuleDialog extends TitleAreaDialog implements IConfigModule
      */
     @Override
     public String getMavenURI() {
+    	if (useCustom && customURI != null) {
+    		return customURI;
+    	}
         return defaultURI;
     }
 

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/ui/startup/ShareMavenArtifactsOnStartup.java

@@ -121,7 +121,7 @@ public class ShareMavenArtifactsOnStartup extends ShareLibrareisHelper {
                 MavenUrlHelper.MVN_PROTOCOL + MavenConstants.LOCAL_RESOLUTION_URL + MavenUrlHelper.REPO_SEPERATOR);
         File file = null;
         try {
-            file = TalendMavenResolver.getMavenResolver().resolve(localMvnUrl);
+            file = TalendMavenResolver.resolve(localMvnUrl);
         } catch (IOException | RuntimeException e) {
             ExceptionHandler.process(e);
         }

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/utils/nexus/NexusDownloadHelperWithProgress.java

@@ -98,23 +98,7 @@ public class NexusDownloadHelperWithProgress extends DownloadHelperWithProgress
     private File resolveJar(ArtifactRepositoryBean customNexusServer, String decryptedMvnUri) throws Exception {
         ILibraryManagerService libManager = (ILibraryManagerService) GlobalServiceRegister.getDefault()
                 .getService(ILibraryManagerService.class);
-        File resolved = null;
-        try {
-            resolved = libManager.resolveJar(customNexusServer, decryptedMvnUri);
-        } catch (Exception e) {
-            // hide the user/password in the error
-            String regex = "\\://(.+)\\:(.+)@";
-            String message = e.getMessage();
-            message = message.replaceAll(regex, "://");
-            Exception cause = null;
-            if (e.getCause() != null) {
-                String causeMessage = e.getCause().getMessage();
-                causeMessage = causeMessage.replaceAll(regex, "://");
-                cause = new Exception(causeMessage);
-            }
-            throw new Exception(message, cause);
-        }
-        return resolved;
+        return libManager.resolveJar(customNexusServer, decryptedMvnUri);
     }
 
     /*

main/plugins/org.talend.librariesmanager/resources/java/routines/Mathematical.java

@@ -5,6 +5,8 @@
 // ============================================================================
 package routines;
 
+import routines.system.RandomUtils;
+
 public class Mathematical {
 
     /**
@@ -284,7 +286,7 @@ public class Mathematical {
      *
      */
     public static double LN(double a) {
-        return Math.log(a) / Math.E;
+        return Math.log(a);
     }
 
     /**
@@ -381,7 +383,7 @@ public class Mathematical {
      *
      */
     public static double RND(double a) {
-        return Math.random() * a;
+        return RandomUtils.random() * a;
     }
 
     /**

main/plugins/org.talend.librariesmanager/resources/java/routines/Numeric.java

@@ -6,10 +6,13 @@
 package routines;
 
 import java.math.BigDecimal;
+import java.util.concurrent.ConcurrentHashMap;
+
+import routines.system.RandomUtils;
 
 public class Numeric {
 
-    private static final java.util.Map<String, Integer> seq_Hash = new java.util.HashMap<String, Integer>();
+    private static final java.util.Map<String, Integer> seq_Hash = new ConcurrentHashMap<>();
 
     /**
      * return an incremented numeric id
@@ -30,13 +33,8 @@ public class Numeric {
      *
      */
     public static Integer sequence(String seqName, int startValue, int step) {
-        if (seq_Hash.containsKey(seqName)) {
-            seq_Hash.put(seqName, seq_Hash.get(seqName) + step);
-            return seq_Hash.get(seqName);
-        } else {
-            seq_Hash.put(seqName, startValue);
-            return startValue;
-        }
+        return seq_Hash.compute(seqName,
+                (String k,  Integer v) -> v == null ? startValue : v + step);
     }
 
     /**
@@ -68,9 +66,7 @@ public class Numeric {
      */
 
     public static void removeSequence(String seqName) {
-        if (seq_Hash.containsKey(seqName)) {
-            seq_Hash.remove(seqName);
-        }
+        seq_Hash.remove(seqName);
     }
 
     /**
@@ -94,7 +90,7 @@ public class Numeric {
         if (max < min) {
             throw new RuntimeException("Max value should be bigger than min value");
         }
-        return ((Long) Math.round(min - 0.5 + (Math.random() * (max - min + 1)))).intValue();
+        return ((Long) Math.round(min - 0.5 + (RandomUtils.random() * (max - min + 1)))).intValue();
     }
 
     /**

main/plugins/org.talend.librariesmanager/resources/java/routines/TalendDataGenerator.java

@@ -5,6 +5,8 @@
 // ============================================================================
 package routines;
 
+import routines.system.RandomUtils;
+
 public class TalendDataGenerator {
 
     /**
@@ -18,7 +20,7 @@ public class TalendDataGenerator {
         String[] list = { "Abraham", "Andrew", "Benjamin", "Bill", "Calvin", "Chester", "Dwight", "Franklin", "George", "Gerald", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$ //$NON-NLS-9$ //$NON-NLS-10$
                 "Grover", "Harry", "Herbert", "James", "Jimmy", "John", "Lyndon", "Martin", "Millard", "Richard", "Ronald", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$ //$NON-NLS-9$ //$NON-NLS-10$ //$NON-NLS-11$
                 "Rutherford", "Theodore", "Thomas", "Ulysses", "Warren", "William", "Woodrow", "Zachary" }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$
-        Integer random = 0 + ((Long) Math.round(Math.random() * (list.length - 1 - 0))).intValue();
+        Integer random = 0 + ((Long) Math.round(RandomUtils.random() * (list.length - 1 - 0))).intValue();
         return list[random];
     }
 
@@ -35,7 +37,7 @@ public class TalendDataGenerator {
                 "Madison", "Monroe", "Carter", "Adams", "Kennedy", "Quincy", "Adams", "Tyler", "Johnson", "Van Buren", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$ //$NON-NLS-9$ //$NON-NLS-10$
                 "Fillmore", "Nixon", "Reagan", "Hayes", "Roosevelt", "Jefferson", "Grant", "Harding", "Harrison", "Taft", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$ //$NON-NLS-9$ //$NON-NLS-10$
                 "McKinley", "Wilson", "Taylor" }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-        Integer random = 0 + ((Long) Math.round(Math.random() * (list.length - 1 - 0))).intValue();
+        Integer random = 0 + ((Long) Math.round(RandomUtils.random() * (list.length - 1 - 0))).intValue();
         return list[random];
     }
 
@@ -60,7 +62,7 @@ public class TalendDataGenerator {
                 "San Simeon", "San Ysidro Blvd", "Santa Ana Freeway", "Santa Monica Road", "Santa Rosa North", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
                 "Santa Rosa South", "South Highway", "South Roosevelt Drive", "Steele Lane", "Tanger Blvd", "Timberlane Drive", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$
                 "Tully Road East", "Via Real", "W. Russell St.", "Westside Freeway", "Woodson Rd." }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
-        Integer random = 0 + ((Long) Math.round(Math.random() * (list.length - 1 - 0))).intValue();
+        Integer random = 0 + ((Long) Math.round(RandomUtils.random() * (list.length - 1 - 0))).intValue();
         return list[random];
     }
 
@@ -78,7 +80,7 @@ public class TalendDataGenerator {
                 "Saint Paul", "Jackson", "Jefferson City", "Helena", "Lincoln", "Carson City", "Concord", "Trenton", "Santa Fe", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$ //$NON-NLS-9$
                 "Albany", "Columbus", "Oklahoma City", "Salem", "Harrisburg", "Providence", "Nashville", "Austin", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$
                 "Salt Lake City", "Montpelier", "Richmond", "Charleston", "Olympia", "Madison", "Cheyenne" }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$
-        Integer random = 0 + ((Long) Math.round(Math.random() * (list.length - 1 - 0))).intValue();
+        Integer random = 0 + ((Long) Math.round(RandomUtils.random() * (list.length - 1 - 0))).intValue();
         return list[random];
     }
 
@@ -96,7 +98,7 @@ public class TalendDataGenerator {
                 "Michigan", "Minnesota", "Mississippi", "Missouri", "Montana", "Nebraska", "Nevada", "New Hampshire", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$
                 "New Jersey", "New Mexico", "New York", "Ohio", "Oklahoma", "Oregon", "Pennsylvania", "Rhode Island", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$
                 "Tennessee", "Texas", "Utah", "Vermont", "Virginia", "West Virginia", "Washington", "Wisconsin", "Wyoming" }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$ //$NON-NLS-9$
-        Integer random = 0 + ((Long) Math.round(Math.random() * (list.length - 1 - 0))).intValue();
+        Integer random = 0 + ((Long) Math.round(RandomUtils.random() * (list.length - 1 - 0))).intValue();
         return list[random];
     }
 
@@ -111,7 +113,7 @@ public class TalendDataGenerator {
         String[] list = { "AL", "AK", "AZ", "AR", "CA", "NC", "SC", "CO", "CT", "ND", "SD", "DE", "FL", "GA", "HI", "ID", "IL", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$ //$NON-NLS-9$ //$NON-NLS-10$ //$NON-NLS-11$ //$NON-NLS-12$ //$NON-NLS-13$ //$NON-NLS-14$ //$NON-NLS-15$ //$NON-NLS-16$ //$NON-NLS-17$
                 "IN", "IA", "KS", "KY", "LA", "ME", "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$ //$NON-NLS-9$ //$NON-NLS-10$ //$NON-NLS-11$ //$NON-NLS-12$ //$NON-NLS-13$ //$NON-NLS-14$ //$NON-NLS-15$ //$NON-NLS-16$ //$NON-NLS-17$ //$NON-NLS-18$ //$NON-NLS-19$
                 "OH", "OK", "OR", "PA", "RI", "TN", "TX", "UT", "VT", "VA", "WV", "WA", "WI", "WY" }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$ //$NON-NLS-9$ //$NON-NLS-10$ //$NON-NLS-11$ //$NON-NLS-12$ //$NON-NLS-13$ //$NON-NLS-14$
-        Integer random = 0 + ((Long) Math.round(Math.random() * (list.length - 1 - 0))).intValue();
+        Integer random = 0 + ((Long) Math.round(RandomUtils.random() * (list.length - 1 - 0))).intValue();
         return list[random];
     }
 }

main/plugins/org.talend.librariesmanager/resources/java/routines/TalendDate.java

@@ -19,6 +19,7 @@ import java.util.TimeZone;
 
 import routines.system.FastDateParser;
 import routines.system.LocaleProvider;
+import routines.system.RandomUtils;
 import routines.system.TalendTimestampWithTZ;
 
 public class TalendDate {
@@ -1147,7 +1148,7 @@ public class TalendDate {
         maxCal.set(Calendar.DAY_OF_MONTH, maxDay);
 
         long random = minCal.getTimeInMillis()
-                + (long) ((maxCal.getTimeInMillis() - minCal.getTimeInMillis() + 1) * Math.random());
+                + (long) ((maxCal.getTimeInMillis() - minCal.getTimeInMillis() + 1) * RandomUtils.random());
         return new Date(random);
     }
 

main/plugins/org.talend.librariesmanager/resources/java/routines/system/GlobalResource.java

@@ -14,28 +14,23 @@ package routines.system;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 public class GlobalResource {
 
-    // let it support the top level Object
-    public static Map<Object, Object> resourceMap = new HashMap<Object, Object>();
+	// let it support the top level Object
+	public static Map<Object, Object> resourceMap = new HashMap<Object, Object>();
 
-    // when there is multiple threads wants to insert stats&logs&meta into DB, it is used as a locker. bug:22677
-    public static TalendMultiThreadLockMap resourceLockMap = new TalendMultiThreadLockMap();
+	// when there is multiple threads wants to insert stats&logs&meta into DB, it is
+	// used as a locker. bug:22677
+	public static TalendMultiThreadLockMap resourceLockMap = new TalendMultiThreadLockMap();
 
-    public static class TalendMultiThreadLockMap {
+	public static class TalendMultiThreadLockMap {
 
-        private Map<Object, Object> tMultiTheadLockMap = new HashMap<Object, Object>();
+		private Map<Object, Object> tMultiTheadLockMap = new ConcurrentHashMap<>();
 
-        public Object get(Object key) {
-            if (tMultiTheadLockMap.get(key) == null) {
-                synchronized (TalendMultiThreadLockMap.this) {
-                    if (tMultiTheadLockMap.get(key) == null) {
-                        tMultiTheadLockMap.put(key, new Object());
-                    }
-                }
-            }
-            return tMultiTheadLockMap.get(key);
-        }
-    }
+		public Object get(Object key) {
+			return tMultiTheadLockMap.computeIfAbsent(key, k -> new Object());
+		}
+	}
 }

main/plugins/org.talend.librariesmanager/resources/java/routines/system/JobStructureCatcherUtils.java

@@ -129,7 +129,7 @@ public class JobStructureCatcherUtils {
 		this.job_version = jobVersion;
 	}
 
-	public void addMessage(String component_id, String component_name, Map<String, String> component_parameters,
+	private void addMessage(String component_id, String component_name, Map<String, String> component_parameters,
 			List<Map<String, String>> component_schema, String input_connectors, String output_connectors,
 			Map<String, String> connector_name_2_connector_schema, boolean current_connector_as_input,
 			String current_connector_type, String current_connector, String currrent_row_content, long row_count,
@@ -149,7 +149,7 @@ public class JobStructureCatcherUtils {
 				current_connector_type, current_connector, null, 0, total_row_number, start_time, end_time, null);
 	}
 
-	public void addComponentMessage(String component_id, String component_name) {
+	public void addCM(String component_id, String component_name) {
 		this.addMessage(component_id, component_name, null, null, null, null, null, false, null, null,
 				null, 0, 0, 0, 0, null);
 	}

main/plugins/org.talend.librariesmanager/resources/java/routines/system/PasswordEncryptUtil.java

@@ -21,21 +21,24 @@ import org.talend.daikon.crypto.KeySources;
  */
 public class PasswordEncryptUtil {
 
-    public static String ENCRYPT_KEY = "Encrypt"; //$NON-NLS-1$
+    public static final String ENCRYPT_KEY = "Encrypt"; //$NON-NLS-1$
 
     private static final String ENCRYPTION_KEY = "Talend_TalendKey";
     
-    public static String PREFIX_PASSWORD = "ENC:["; //$NON-NLS-1$
+    private static final String PREFIX_PASSWORD = "ENC:["; //$NON-NLS-1$
     
-    public static String POSTFIX_PASSWORD = "]"; //$NON-NLS-1$
+    private static final String POSTFIX_PASSWORD = "]"; //$NON-NLS-1$
 
-    private static Encryption defaultEncryption;
+    private static final Encryption ENCRYPTION = new Encryption(KeySources.fixedKey(ENCRYPTION_KEY), CipherSources.getDefault());
+
+    private PasswordEncryptUtil() {
+    }
 
     public static String encryptPassword(String input) throws Exception {
         if (input == null) {
             return input;
         }
-        return PREFIX_PASSWORD + getEncryption().encrypt(input) + POSTFIX_PASSWORD;
+        return PREFIX_PASSWORD + ENCRYPTION.encrypt(input) + POSTFIX_PASSWORD;
     }
 
     public static String decryptPassword(String input) {
@@ -44,7 +47,7 @@ public class PasswordEncryptUtil {
         }
         if (input.startsWith(PREFIX_PASSWORD) && input.endsWith(POSTFIX_PASSWORD)) {
             try {
-                return getEncryption()
+                return ENCRYPTION
                         .decrypt(input.substring(PREFIX_PASSWORD.length(), input.length() - POSTFIX_PASSWORD.length()));
             } catch (Exception e) {
                 // do nothing
@@ -53,13 +56,6 @@ public class PasswordEncryptUtil {
         return input;
     }
 
-    private static Encryption getEncryption() {
-        if (defaultEncryption == null) {
-            defaultEncryption = new Encryption(KeySources.fixedKey(ENCRYPTION_KEY), CipherSources.aes());
-        }
-        return defaultEncryption;
-    }
-
     public static final String PASSWORD_FOR_LOGS_VALUE = "...";
 
 }

main/plugins/org.talend.librariesmanager/resources/java/routines/system/RandomUtils.java

@@ -0,0 +1,15 @@
+package routines.system;
+
+import java.security.SecureRandom;
+
+public class RandomUtils {
+
+	//lazy init, but no meaning now as only one method
+	private static final class RandomNumberGeneratorHolder {
+		static final SecureRandom randomNumberGenerator = new SecureRandom();
+	}
+
+	public static double random() {
+		return RandomNumberGeneratorHolder.randomNumberGenerator.nextDouble();
+	}
+}

main/plugins/org.talend.librariesmanager/resources/java/routines/system/RunStat.java

@@ -20,6 +20,7 @@ import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 public class RunStat implements Runnable {
 
@@ -51,7 +52,7 @@ public class RunStat implements Runnable {
 
     public static String TYPE1_CONNECTION = "1";
 
-    public class StatBean {
+    private class StatBean {
 
         private String itemId;
 
@@ -333,7 +334,7 @@ public class RunStat implements Runnable {
 
     private List<String> keysList4Meter = new LinkedList<String>();
 
-    public synchronized StatBean logStatOnConnection(String connectionId, int mode, int nbLine) {
+    public synchronized StatBean log(String connectionId, int mode, int nbLine) {
         StatBean bean;
         String key = connectionId;
         if (connectionId.contains(".")) {
@@ -386,7 +387,185 @@ public class RunStat implements Runnable {
 
         return bean;
     }
+    
+    public synchronized boolean log(Map<String, Object> resourceMap, String iterateId, String connectionUniqueName, int mode, int nbLine, 
+    		JobStructureCatcherUtils jscu, String sourceNodeId, String sourceNodeComponent, String targetNodeId, String targetNodeComponent, String lineType) {
+    	if(resourceMap.get("inIterateVComp") == null || !((Boolean)resourceMap.get("inIterateVComp"))) {
+	    	StatBean bean = log(connectionUniqueName, mode, nbLine);
+	    	jscu.addConnectionMessage(
+	    		sourceNodeId, 
+	    		sourceNodeComponent, 
+			    false,
+			    lineType,
+			    connectionUniqueName+iterateId,
+			    bean.getNbLine(),
+			    bean.getStartTime(),
+			    bean.getEndTime()
+			);
+			
+	 		jscu.addConnectionMessage(
+				targetNodeId, 
+				targetNodeComponent, 
+			    true,
+			    "input",
+			    connectionUniqueName+iterateId,
+			    bean.getNbLine(),
+			    bean.getStartTime(),
+			    bean.getEndTime()
+			);
+	 		
+	 		return true;
+    	} else {
+    		return false;
+    	}
+    }
 
+    /**
+     * work for avoiding the 65535 issue
+     */
+    public synchronized void updateStat(Map<String, Object> resourceMap, String iterateId, int mode, int nbLine, String... connectionUniqueNames) {
+    	if(resourceMap.get("inIterateVComp") == null || !((Boolean)resourceMap.get("inIterateVComp"))){
+	    	for(String connectionUniqueName : connectionUniqueNames) {
+	    		updateStatOnConnection(connectionUniqueName+iterateId, mode, nbLine);
+	    	}
+    	}
+    }
+    
+    /**
+     * work for avoiding the 65535 issue
+     */
+    public synchronized boolean updateStatAndLog(boolean execStat, boolean enableLogStash, Map<String, Object> resourceMap, String iterateId, String connectionUniqueName, int mode, int nbLine, 
+    		JobStructureCatcherUtils jscu, String sourceNodeId, String sourceNodeComponent, String targetNodeId, String targetNodeComponent, String lineType) {
+    	if(execStat) {
+    		updateStat(resourceMap, iterateId, mode, nbLine, connectionUniqueName);
+    	}
+    	
+    	if(enableLogStash) {
+    		return log(resourceMap, iterateId, connectionUniqueName, mode, nbLine, 
+    	    		jscu, sourceNodeId, sourceNodeComponent, targetNodeId, targetNodeComponent, lineType);
+    	}
+    	
+    	
+    	return false;
+    }
+    
+    /**
+     * work for avoiding the 65535 issue
+     */
+    public synchronized void updateStatOnConnection(Map<String, Object> resourceMap, String iterateId, int mode, int nbLine, String... connectionUniqueNames) {
+    	if(resourceMap.get("inIterateVComp") == null){
+	    	for(String connectionUniqueName : connectionUniqueNames) {
+	    		updateStatOnConnection(connectionUniqueName+iterateId, mode, nbLine);
+	    	}
+    	}
+    }
+    
+    /**
+     * work for avoiding the 65535 issue
+     */
+    public synchronized void log(Map<String, Object> resourceMap, String iterateId, int mode, int nbLine, String... connectionUniqueNames) {
+    	if(resourceMap.get("inIterateVComp") == null){
+	    	for(String connectionUniqueName : connectionUniqueNames) {
+	    		log(connectionUniqueName+iterateId, mode, nbLine);
+	    	}
+    	}
+    }
+    
+    /**
+     * work for avoiding the 65535 issue
+     */
+    public synchronized void updateStatAndLog(boolean execStat, boolean enableLogStash, Map<String, Object> resourceMap, String iterateId, int mode, int nbLine, String... connectionUniqueNames) {
+    	if(execStat) {
+    		updateStatOnConnection(resourceMap, iterateId, mode, nbLine, connectionUniqueNames);
+    	}
+    	
+    	if(enableLogStash) {
+    		log(resourceMap, iterateId, mode, nbLine, connectionUniqueNames);
+    	}
+    }
+    
+    /**
+     * work for avoiding the 65535 issue
+     */
+    public synchronized void updateStatOnConnection(String iterateId, int mode, int nbLine, String... connectionUniqueNames) {
+    	for(String connectionUniqueName : connectionUniqueNames) {
+    		updateStatOnConnection(connectionUniqueName+iterateId, mode, nbLine);
+    	}
+    }
+    
+    /**
+     * work for avoiding the 65535 issue
+     */
+    public synchronized void log(String iterateId, int mode, int nbLine, String... connectionUniqueNames) {
+    	for(String connectionUniqueName : connectionUniqueNames) {
+    		log(connectionUniqueName+iterateId, mode, nbLine);
+    	}
+    }
+    
+    /**
+     * TBD-9420 fix 
+     */
+    public synchronized void log(String iterateId, int mode, int nbLine, String connectionUniqueName) {
+		log(connectionUniqueName+iterateId, mode, nbLine);
+    }
+    
+    /**
+     * work for avoiding the 65535 issue
+     */
+    public synchronized void updateStatAndLog(boolean execStat, boolean enableLogStash, String iterateId, int mode, int nbLine, String... connectionUniqueNames) {
+    	if(execStat) {
+    		updateStatOnConnection(iterateId, mode, nbLine, connectionUniqueNames);
+    	}
+    	
+    	if(enableLogStash) {
+    		log(iterateId, mode, nbLine, connectionUniqueNames);
+    	}
+    }
+    
+    /**
+     * work for avoiding the 65535 issue
+     */
+    public synchronized void updateStatOnConnectionAndLog(Map<String, Object> globalMap, int iterateLoop, String iterateId, boolean execStat, boolean enableLogStash, int nbLine, String... connectionUniqueNames) {
+    	for(String connectionUniqueName : connectionUniqueNames) {
+	    	ConcurrentHashMap<Object, Object> concurrentHashMap = (ConcurrentHashMap) globalMap.get("concurrentHashMap");
+			concurrentHashMap.putIfAbsent(connectionUniqueName + iterateLoop,new java.util.concurrent.atomic.AtomicInteger(0));
+			java.util.concurrent.atomic.AtomicInteger stats = (java.util.concurrent.atomic.AtomicInteger) concurrentHashMap.get(connectionUniqueName + iterateLoop);
+			
+			int step = stats.incrementAndGet()<=1?0:1;
+			
+			if(execStat) {
+				updateStatOnConnection(connectionUniqueName+iterateId, step, nbLine);
+			}
+			
+			if(enableLogStash) {
+				log(connectionUniqueName+iterateId, step, nbLine);
+			}
+    	}
+    }
+    
+    /**
+     * work for avoiding the 65535 issue
+     */
+    public synchronized void updateStatOnConnectionAndLog(Map<String, Object> resourceMap, Map<String, Object> globalMap, int iterateLoop, String iterateId, boolean execStat, boolean enableLogStash, int nbLine, String... connectionUniqueNames) {
+    	for(String connectionUniqueName : connectionUniqueNames) {
+    		if(resourceMap.get("inIterateVComp") == null) {
+		    	ConcurrentHashMap<Object, Object> concurrentHashMap = (ConcurrentHashMap) globalMap.get("concurrentHashMap");
+				concurrentHashMap.putIfAbsent(connectionUniqueName + iterateLoop,new java.util.concurrent.atomic.AtomicInteger(0));
+				java.util.concurrent.atomic.AtomicInteger stats = (java.util.concurrent.atomic.AtomicInteger) concurrentHashMap.get(connectionUniqueName + iterateLoop);
+				
+				int step = stats.incrementAndGet()<=1?0:1;
+				
+				if(execStat) {
+					updateStatOnConnection(connectionUniqueName+iterateId, step, nbLine);
+				}
+				
+				if(enableLogStash) {
+					log(connectionUniqueName+iterateId, step, nbLine);
+				}
+    		}
+    	}
+    }
+    
     public synchronized void updateStatOnConnection(String connectionId, int mode, int nbLine) {
         StatBean bean;
         String key = connectionId;

main/plugins/org.talend.librariesmanager/resources/java/routines/system/StringUtils.java

@@ -403,8 +403,25 @@ public class StringUtils {
 
         return sb.toString();
     }
-
-
+    
+    /**
+     * check if string contains search string case-insensitivity
+     * the code is copied from apache commons-lang3 StringUtils and do some adjust to avoid the useless code
+     */
+    public static boolean containsIgnoreCase(final String str, final String searchStr) {
+        if (str == null || searchStr == null) {
+            return false;
+        }
+        final int len = searchStr.length();
+        final int max = str.length() - len;
+        for (int i = 0; i <= max; i++) {
+            if (str.regionMatches(true, i, searchStr, 0, len)) {
+                return true;
+            }
+        }
+        return false;
+    }
+    
     /**
      * return null value not "null" String when obj is null that is the only difference with String.valueOf(Object obj)
      *

main/plugins/org.talend.librariesmanager/resources/java/sqltemplates/Snowflake/SnowflakeCreateSourceTable.sqltemplate

@@ -0,0 +1,41 @@
+<%
+EXTRACT(__SCHEMA__);
+String fieldsCreated = "";
+boolean flagCreated=false;
+
+for(int i=0; i < __SCHEMA__.length; i++){
+ if(flagCreated){
+	 fieldsCreated += ",";
+ }else{
+	 flagCreated=true;
+ }
+ 	if( __SCHEMA_DBTYPE__[i].equals("VARCHAR") || __SCHEMA_DBTYPE__[i].equals("VARCHAR2") || __SCHEMA_DBTYPE__[i].equals("CHAR") || __SCHEMA_DBTYPE__[i].equals("BYTE") || __SCHEMA_DBTYPE__[i].equals("VARBYTE")){
+ 		fieldsCreated += (__SCHEMA_NAME__[i] + " " + __SCHEMA_DBTYPE__[i] +" ("+ __SCHEMA_LENGTH__[i] +") ");
+ 	} else if(__SCHEMA_DBTYPE__[i].equals("NUMBER") || __SCHEMA_DBTYPE__[i].equals("DECIMAL")){
+ 		fieldsCreated += (__SCHEMA_NAME__[i] + " " + __SCHEMA_DBTYPE__[i] +" ("+ __SCHEMA_LENGTH__[i] +","+ __SCHEMA_PRECISION__[i] +") ");
+ 	} else {
+ 		fieldsCreated += (__SCHEMA_NAME__[i] + " " + __SCHEMA_DBTYPE__[i]);
+    }
+ 	if(__SCHEMA_NULLABLE__[i].equals("false")){
+ 		fieldsCreated += " NOT NULL";
+ 	}
+}
+String primary = "";
+for(int j=0; j < __SCHEMA__.length; j++){
+	if(__SCHEMA_ISKEY__[j].equals("true") && __SCHEMA_NULLABLE__[j].equals("false")){
+		if(primary.startsWith(" , PRIMARY KEY")) {
+			primary += ","+__SCHEMA_NAME__[j];
+		} else {
+			primary += " , PRIMARY KEY ("+__SCHEMA_NAME__[j];
+		}
+	}
+}
+if(!primary.equals("")){
+	fieldsCreated += primary+")";
+}
+%>
+
+CREATE TABLE <%= __TABLE_NAME__ %>
+(
+<%= fieldsCreated %>
+);

main/plugins/org.talend.librariesmanager/resources/java/sqltemplates/Snowflake/SnowflakeCreateTargetTable.sqltemplate

@@ -0,0 +1,41 @@
+<%
+EXTRACT(__SCHEMA_TARGET__);
+String fieldsCreated = "";
+boolean flagCreated=false;
+
+for(int i=0; i < __SCHEMA_TARGET__.length; i++){
+ if(flagCreated){
+	 fieldsCreated += ",";
+ }else{
+	 flagCreated=true;
+ }
+ 	if( __SCHEMA_TARGET_DBTYPE__[i].equals("VARCHAR") || __SCHEMA_TARGET_DBTYPE__[i].equals("VARCHAR2") || __SCHEMA_TARGET_DBTYPE__[i].equals("CHAR") || __SCHEMA_TARGET_DBTYPE__[i].equals("BYTE") || __SCHEMA_TARGET_DBTYPE__[i].equals("VARBYTE")){
+ 		fieldsCreated += (__SCHEMA_TARGET_NAME__[i] + " " + __SCHEMA_TARGET_DBTYPE__[i] +" ("+ __SCHEMA_TARGET_LENGTH__[i] +") ");
+ 	} else if(__SCHEMA_TARGET_DBTYPE__[i].equals("NUMBER") || __SCHEMA_TARGET_DBTYPE__[i].equals("DECIMAL")){
+ 		fieldsCreated += (__SCHEMA_TARGET_NAME__[i] + " " + __SCHEMA_TARGET_DBTYPE__[i] +" ("+ __SCHEMA_TARGET_LENGTH__[i] +","+ __SCHEMA_TARGET_PRECISION__[i] +") ");
+ 	} else {
+ 		fieldsCreated += (__SCHEMA_TARGET_NAME__[i] + " " + __SCHEMA_TARGET_DBTYPE__[i]);
+    }
+ 	if(__SCHEMA_TARGET_NULLABLE__[i].equals("false")){
+ 		fieldsCreated += " NOT NULL";
+ 	}
+}
+String primary = "";
+for(int j=0; j < __SCHEMA_TARGET__.length; j++){
+	if(__SCHEMA_TARGET_ISKEY__[j].equals("true") && __SCHEMA_TARGET_NULLABLE__[j].equals("false")){
+		if(primary.startsWith(" , PRIMARY KEY")) {
+			primary += ","+__SCHEMA_TARGET_NAME__[j];
+		} else {
+			primary += " , PRIMARY KEY ("+__SCHEMA_TARGET_NAME__[j];
+		}
+	}
+}
+if(!primary.equals("")){
+	fieldsCreated += primary+")";
+}
+%>
+
+CREATE TABLE <%= __TABLE_NAME_TARGET__ %>
+(
+<%= fieldsCreated %>
+);

main/plugins/org.talend.librariesmanager/resources/java/sqltemplates/Snowflake/SnowflakeDropSourceTable.sqltemplate

@@ -0,0 +1,2 @@
+-- DROP SOURCE TABLE
+DROP TABLE <%= __TABLE_NAME__ %>;

main/plugins/org.talend.librariesmanager/resources/java/sqltemplates/Snowflake/SnowflakeDropTargetTable.sqltemplate

@@ -0,0 +1,2 @@
+-- DROP TARGET TABLE
+DROP TABLE <%=__TABLE_NAME_TARGET__%>;

main/plugins/org.talend.librariesmanager/resources/java/sqltemplates/Snowflake/SnowflakeMerge.sqltemplate

@@ -0,0 +1,101 @@
+-- This pattern generates a merge statement for Snowflake
+--(c) 2019 Talend Inc.
+<%
+	EXTRACT(__MERGE_ON__);
+	EXTRACT(__UPDATE_MAPPING__);
+	EXTRACT(__ADDITIONAL_UPDATE_COLUMNS__);
+	EXTRACT(__INSERT_MAPPING__);
+	EXTRACT(__ADDITIONAL_INSERT_COLUMNS__);
+	String dbType = "__DBTYPE__";
+	boolean hasUpdateClause = __USE_UPDATE__;
+	boolean hasAddtionalUpdateColumns = __SPECIFY_ADDITIONAL_UPDATE_COLUMNS__;
+	boolean hasUpdateWhereClause = __SPECIFY_UPDATE_WHERE_CLAUSE__;
+	boolean hasInsertClause = __USE_INSERT__;
+	boolean hasAdditionalInsertColumns = __SPECIFY_ADDITIONAL_INSERT_COLUMNS__;
+	boolean hasInsertWhereClause = __SPECIFY_INSERT_WHERE_CLAUSE__;
+
+	boolean flag = false;
+	String mergeOnStr = "";
+	for(int i=0; i < __MERGE_ON_SRC_COLUMN__.length; i++){
+		if (flag) {
+			mergeOnStr += " AND ";
+		} else {
+			flag = true;
+		}
+		mergeOnStr += (__TARGET_TABLE__ + "." + __MERGE_ON_TRG_COLUMN__[i] + " = " + __SOURCE_TABLE__ + "." + __MERGE_ON_SRC_COLUMN__[i]);
+	}
+
+	String updateStr = "";
+	String updateWhereStr = "";
+	flag = false;
+	if (hasUpdateClause) {
+		for(int i=0; i < __UPDATE_MAPPING_SRC_COLUMN__.length; i++) {
+			if (flag) {
+				updateStr += ",";
+			} else {
+				flag = true;
+			}
+			updateStr += (__TARGET_TABLE__ + "." + __UPDATE_MAPPING_TRG_COLUMN__[i] + " = " + __SOURCE_TABLE__ + "." + __UPDATE_MAPPING_SRC_COLUMN__[i]);
+		}
+		if (hasAddtionalUpdateColumns) {
+			for(int i=0; i < __ADDITIONAL_UPDATE_COLUMNS_TRG_COLUMN_VALUE__.length; i++) {
+				if (flag) {
+					updateStr += ",";
+				} else {
+					flag = true;
+				}
+				updateStr += (__TARGET_TABLE__ + "." + __ADDITIONAL_UPDATE_COLUMNS_TRG_COLUMN_NAME__[i] + " = " + __ADDITIONAL_UPDATE_COLUMNS_TRG_COLUMN_VALUE__[i]);
+			}
+		}
+		if (hasUpdateWhereClause) {
+			updateWhereStr = " " + __UPDATE_WHERE_CLAUSE__ + " ";
+		}
+	}
+
+	String insertColumnsStr = "";
+	String insertValuesStr = "";
+	String insertWhereStr = "";
+	String insertStr = "";
+	flag = false;
+	if (hasInsertClause) {
+		for(int i=0; i < __INSERT_MAPPING_TRG_COLUMN__.length; i++) {
+			if (flag) {
+				insertColumnsStr += ",";
+				insertValuesStr += ",";
+			} else {
+				flag = true;
+			}
+			insertColumnsStr += (__TARGET_TABLE__ + "." + __INSERT_MAPPING_TRG_COLUMN__[i]);
+			insertValuesStr += (__SOURCE_TABLE__ + "." + __INSERT_MAPPING_SRC_COLUMN__[i]);
+		}
+		if (hasAdditionalInsertColumns) {
+			for(int i=0; i < __ADDITIONAL_INSERT_COLUMNS_TRG_COLUMN_NAME__.length; i++) {
+				if (flag) {
+					insertColumnsStr += ",";
+					insertValuesStr += ",";
+				} else {
+					flag = true;
+				}
+				insertColumnsStr += (__TARGET_TABLE__ + "." + __ADDITIONAL_INSERT_COLUMNS_TRG_COLUMN_NAME__[i]);
+				insertValuesStr += (__ADDITIONAL_INSERT_COLUMNS_TRG_COLUMN_VALUE__[i]);
+			}
+		}
+		if (hasInsertWhereClause) {
+			insertWhereStr = " " + __INSERT_WHERE_CLAUSE__ + " ";
+		}
+
+		insertStr = "(" + insertColumnsStr + ") VALUES (" + insertValuesStr + ")";
+	}
+
+	String mergeString = "";
+
+	if (hasInsertClause || hasUpdateClause) {
+		mergeString = "MERGE INTO " + __TARGET_TABLE__ + " USING " + __SOURCE_TABLE__ +
+					  " ON (" + mergeOnStr + ")" +
+					  (hasUpdateClause ? (" WHEN MATCHED "     + (hasUpdateWhereClause ? ("AND " + updateWhereStr) : "") + " THEN UPDATE SET " + updateStr) : "") +
+					  (hasInsertClause ? (" WHEN NOT MATCHED " + (hasInsertWhereClause ? ("AND " + insertWhereStr) : "") + " THEN INSERT "     + insertStr) : "");
+	} else {
+		mergeString = "DELETE FROM " + __TARGET_TABLE__ + " WHERE 1=0";
+	}
+%>
+<%=mergeString%>;

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/ModulesNeededProvider.java

@@ -706,8 +706,13 @@ public class ModulesNeededProvider {
                 ModuleNeeded toAdd = new ModuleNeeded(context, currentImport.getMODULE(), currentImport.getMESSAGE(),
                         isRequired);
                 toAdd.setMavenUri(currentImport.getMVN());
-                if (!isRequired && "BeanItem".equals(routine.eClass().getName())) {
-                	toAdd.getExtraAttributes().put("IS_OSGI_EXCLUDED", Boolean.TRUE);
+                if (!isRequired) {
+                    if ("BeanItem".equals(routine.eClass().getName())) {
+                        toAdd.getExtraAttributes().put("IS_OSGI_EXCLUDED", Boolean.TRUE);
+                    }
+                    if ("RoutineItem".equals(routine.eClass().getName())) {
+                        continue;
+                    }
                 }
                 // toAdd.setStatus(ELibraryInstallStatus.INSTALLED);
                 importNeedsList.add(toAdd);

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/LocalLibraryManager.java

@@ -46,7 +46,6 @@ import org.eclipse.core.runtime.preferences.ConfigurationScope;
 import org.eclipse.core.runtime.preferences.IEclipsePreferences;
 import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.eclipse.emf.common.util.EMap;
-import org.ops4j.pax.url.mvn.MavenResolver;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.FrameworkUtil;
 import org.osgi.framework.ServiceReference;
@@ -300,6 +299,17 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         return retrieve(testModule, pathToStore, popUp, refresh);
     }
 
+	@Override
+	public boolean retrieve(String jarNeeded, String mavenUri, String pathToStore, IProgressMonitor... monitorWrap) {
+		ModuleNeeded testModule = new ModuleNeeded("", jarNeeded, "", true);
+		if(mavenUri != null) {
+			testModule.setMavenUri(mavenUri);
+		}
+        boolean refresh = true;
+        return retrieve(testModule, pathToStore, true, refresh);
+	}
+
+
     private boolean retrieve(ModuleNeeded module, String pathToStore, boolean showDialog, boolean refresh) {
         String jarNeeded = module.getModuleName();
         String sourcePath = null;
@@ -971,7 +981,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
             localMavenUri = mvnUriStatusKey.replace("mvn:", "mvn:" + MavenConstants.LOCAL_RESOLUTION_URL + "!"); //$NON-NLS-1$ //$NON-NLS-2$
         }
         try {
-            File resolvedJar = TalendMavenResolver.getMavenResolver().resolve(localMavenUri);
+            File resolvedJar = TalendMavenResolver.resolve(localMavenUri);
             if (resolvedJar != null) {
                 try {
                     updatePomFileForJar(mvnUriStatusKey);
@@ -993,7 +1003,6 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
 
     private void updatePomFileForJar(String mvnUri) throws Exception {
         try {
-            MavenResolver mavenResolver = TalendMavenResolver.getMavenResolver();
             MavenArtifact ma = MavenUrlHelper.parseMvnUrl(mvnUri);
             if (ma != null) {
                 String repositoryUrl = ma.getRepositoryUrl();
@@ -1015,14 +1024,14 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
                     File pomFile = null;
                     Exception pomEx = null;
                     try {
-                        pomFile = mavenResolver.resolve(MavenUrlHelper.generateMvnUrl(pomMa));
+                        pomFile = TalendMavenResolver.resolve(MavenUrlHelper.generateMvnUrl(pomMa));
                     } catch (Exception e) {
                         pomEx = e;
                     }
                     if (pomFile == null && classifier != null && !classifier.trim().isEmpty()) {
                         pomMa.setClassifier(classifier);
                         try {
-                            pomFile = mavenResolver.resolve(MavenUrlHelper.generateMvnUrl(pomMa));
+                            pomFile = TalendMavenResolver.resolve(MavenUrlHelper.generateMvnUrl(pomMa));
                         } catch (Exception e) {
                             // ignore
                         }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/AbstractArtifactRepositoryHandler.java

@@ -13,9 +13,13 @@
 package org.talend.librariesmanager.nexus;
 
 import java.io.File;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.util.Dictionary;
 import java.util.Hashtable;
 
+import org.apache.commons.lang.StringUtils;
 import org.talend.core.nexus.ArtifactRepositoryBean;
 import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.NexusConstants;
@@ -62,10 +66,22 @@ public abstract class AbstractArtifactRepositoryHandler implements IRepositoryAr
         if (props == null) {
             props = new Hashtable<String, String>();
         }
-        String repositories = null;
-        String custom_server = serverBean.getServer();
+
         String custom_user = serverBean.getUserName();
         String custom_pass = serverBean.getPassword();
+        try {
+            if (StringUtils.isNotBlank(custom_user)) {
+                custom_user = URLEncoder.encode(custom_user, StandardCharsets.UTF_8.toString());
+            }
+            if (StringUtils.isNotBlank(custom_pass)) {
+                custom_pass = URLEncoder.encode(custom_pass, StandardCharsets.UTF_8.toString());
+            }
+        } catch (UnsupportedEncodingException e1) {
+            throw new RuntimeException(e1);
+        }
+
+        String repositories = null;
+        String custom_server = serverBean.getServer();
         String release_rep = serverBean.getRepositoryId();
         String snapshot_rep = serverBean.getSnapshotRepId();
         if (custom_server.endsWith(NexusConstants.SLASH)) {

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/utils/ConnectionContextHelper.java

@@ -1672,7 +1672,7 @@ public final class ConnectionContextHelper {
                     }
                 }
                 for (String var : neededVars) {
-                    if (context.getContextParameter(var) != null) {
+                    if (context.containsSameParameterIgnoreCase(var)) {
                         continue;
                     }
                     ContextParameterType param = ContextUtils.getContextParameterTypeByName(type, var);

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/utils/SwitchContextGroupNameImpl.java

@@ -74,6 +74,16 @@ public class SwitchContextGroupNameImpl implements ISwitchContext {
      */
     @Override
     public boolean updateContextGroup(ConnectionItem connItem, String selectedContext) {
+        return updateContextGroup(connItem, selectedContext, null);
+    }
+
+    /*
+     * (non-Javadoc)
+     *
+     * @see org.talend.core.model.metadata.builder.database.ISwitchContext#updateContextGroup(org.talend.core.model.
+     * properties .ContextItem, org.talend.core.model.metadata.builder.connection.Connection)
+     */
+    public boolean updateContextGroup(ConnectionItem connItem, String selectedContext, String originalContext) {
         if (connItem == null) {
             return false;
         }
@@ -81,9 +91,9 @@ public class SwitchContextGroupNameImpl implements ISwitchContext {
         // MOD msjian 2012-2-13 TDQ-4559: make it support file/mdm connection
         if (con != null) {
             // TDQ-4559~
-            String oldContextName = con.getContextName();
+            String oldContextName = originalContext == null ? con.getContextName() : originalContext;
 
-            if (!isContextIsValid(selectedContext, con)) {
+            if (!isContextIsValid(selectedContext, oldContextName, con)) {
                 return false;
             }
             con.setContextName(selectedContext);
@@ -112,8 +122,7 @@ public class SwitchContextGroupNameImpl implements ISwitchContext {
      * @param selectedContext
      * @paramconn
      */
-    private boolean isContextIsValid(String selectedContext, Connection conn) {
-        String oldContextName = conn.getContextName();
+    private boolean isContextIsValid(String selectedContext, String oldContextName, Connection conn) {
         boolean retCode = false;
         if (conn instanceof DatabaseConnection) {
             EDatabaseTypeName dbType = EDatabaseTypeName.getTypeFromDbType(((DatabaseConnection) conn).getDatabaseType());

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/ExtractMetaDataUtils.java

@@ -1042,6 +1042,9 @@ public class ExtractMetaDataUtils {
                         }
                     }else{
                         String jarName = librairesManagerService.getJarNameFromMavenuri(driverJarPathArg);
+                        if (jarName == null) {
+                            jarName = driverJarPathArg.split("/")[1] + ".jar";
+                        }
                         if (!new File(getJavaLibPath() + jarName).exists()) {
                             librairesManagerService.retrieve(jarName, getJavaLibPath(), new NullProgressMonitor());
                         }

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/manager/ExtractManager.java

@@ -165,7 +165,7 @@ public class ExtractManager {
             tablesToFilter = new ArrayList<String>();
         }
         try {
-            Set<String> availableTableTypes = getAvailableTableTypes(dbMetaData);
+            Set<String> availableTableTypes = getTableTypes(dbMetaData);
             retrieveTables(dbMetaData, schema, medataTables, availableTableTypes, tablesToFilter, limit);
         } catch (SQLException e) {
             ExceptionHandler.process(e);
@@ -193,7 +193,7 @@ public class ExtractManager {
         ResultSet rsTableTypes = dbMetaData.getTableTypes();
         Set<String> availableTableTypes = new HashSet<String>();
         String[] neededTableTypes = { ETableTypes.TABLETYPE_TABLE.getName(), ETableTypes.TABLETYPE_VIEW.getName(),
-                ETableTypes.TABLETYPE_SYNONYM.getName() };
+                ETableTypes.TABLETYPE_SYNONYM.getName(), ETableTypes.EXTERNAL_TABLE.getName()};
 
         try {
             while (rsTableTypes.next()) {
@@ -213,6 +213,48 @@ public class ExtractManager {
         }
         return availableTableTypes;
     }
+    
+    public Set<String> getTableTypes(DatabaseMetaData dbMetaData) throws SQLException{
+    	ResultSet rsTableTypes = dbMetaData.getTableTypes();
+        Set<String> availableTableTypes = new HashSet<String>();
+        String[] neededTableTypes = { ETableTypes.TABLETYPE_TABLE.getName(), ETableTypes.TABLETYPE_VIEW.getName(),
+                ETableTypes.TABLETYPE_SYNONYM.getName(), ETableTypes.EXTERNAL_TABLE.getName()};
+        try {
+            while (rsTableTypes.next()) {
+                // StringUtils.trimToEmpty(name) is because bug 4547
+                String currentTableType = StringUtils.trimToEmpty(rsTableTypes.getString(ExtractManager.TABLE_TYPE));
+                // Because BASE TABLE which UniJDBCClientResultSet gets is not the support type of
+                // UniJDBCDatabaseMetaData, do this so as to dispose bug 17281.
+                if ("BASE TABLE".equalsIgnoreCase(currentTableType)) { //$NON-NLS-1$
+                    currentTableType = ETableTypes.TABLETYPE_TABLE.getName();
+                }
+                if (ArrayUtils.contains(neededTableTypes, currentTableType)) {
+                    availableTableTypes.add(currentTableType);
+                }
+            }
+        } finally {
+            rsTableTypes.close();// See bug 5029 Avoid "Invalid cursor exception"
+        }
+        
+        if (EDatabaseTypeName.HIVE.getDisplayName().equals(dbType.getDbType())) {
+        	availableTableTypes.add(ETableTypes.MANAGED_TABLE.getName());
+        	availableTableTypes.add(ETableTypes.INDEX_TABLE.getName());
+        	availableTableTypes.add(ETableTypes.VIRTUAL_VIEW.getName());
+        } else if (EDatabaseTypeName.SAPHana.getDisplayName().equals(dbType.getDbType())) {
+        	availableTableTypes.add(ETableTypes.TABLETYPE_CALCULATION_VIEW.getName());
+        } else if (EDatabaseTypeName.MYSQL.getDisplayName().equals(dbType.getDbType())) {
+        	availableTableTypes.add(ETableTypes.SYSTEM_TABLE.getName());
+        	availableTableTypes.add(ETableTypes.SYSTEM_VIEW.getName());
+        }else if (EDatabaseTypeName.AS400.getDisplayName().equals(dbType.getDbType())) {
+        	availableTableTypes.add(ETableTypes.TABLETYPE_TABLE.getName());
+            availableTableTypes.add(ETableTypes.EXTERNAL_TABLE.getName());
+            availableTableTypes.add(ETableTypes.TABLETYPE_VIEW.getName());
+            availableTableTypes.add(ETableTypes.TABLETYPE_SYNONYM.getName());
+        }else if (MetadataConnectionUtils.isOracle(dbMetaData)) {
+        	availableTableTypes.add(ETableTypes.TABLETYPE_ALL_SYNONYM.getName());
+        }
+        return availableTableTypes;
+    }
 
     public String getSchema(IMetadataConnection metadataConnection) {
         return metadataConnection != null ? metadataConnection.getSchema() : null;

main/plugins/org.talend.model/src/main/java/org/talend/core/model/metadata/builder/connection/Connection.java

@@ -202,5 +202,19 @@ public interface Connection extends AbstractMetadataObject, DataProvider {
      * @generated NOT
      */
     String getValue(String value, boolean encrypt);
+    
+    
+    /**
+     * <p>
+     * Set encryption and decryption functions which will be used inside getValue
+     * </p>
+     * <p>
+     * By default, getValue depends on encrypt/decrypt of {@link org.talend.utils.StudioEncryption}.
+     * </p>
+     * 
+     * @generated NOT
+     */
+    void setEncryptAndDecryptFuncPair(java.util.function.Function<String, String> encrypt,
+            java.util.function.Function<String, String> decrypt);
 
 } // Connection

main/plugins/org.talend.model/src/main/java/org/talend/core/model/metadata/builder/connection/impl/ConnectionImpl.java

@@ -20,7 +20,8 @@ import org.eclipse.emf.ecore.util.InternalEList;
 import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.model.metadata.builder.connection.ConnectionPackage;
 import org.talend.core.model.metadata.builder.connection.QueriesConnection;
-import org.talend.daikon.security.CryptoHelper;
+import org.talend.utils.security.StudioEncryption;
+
 import orgomg.cwm.foundation.softwaredeployment.Component;
 import orgomg.cwm.foundation.softwaredeployment.DataManager;
 import orgomg.cwm.foundation.softwaredeployment.DataProvider;
@@ -292,6 +293,10 @@ public class ConnectionImpl extends AbstractMetadataObjectImpl implements Connec
 
     protected boolean readOnly = false;
 
+    private java.util.function.Function<String, String> encrypt;
+
+    private java.util.function.Function<String, String> decrypt;
+
     /**
      * <!-- begin-user-doc --> <!-- end-user-doc -->
      * @generated
@@ -1196,11 +1201,24 @@ public class ConnectionImpl extends AbstractMetadataObjectImpl implements Connec
      */
     public String getValue(String value, boolean encrypt) {
         if (!isContextMode() && value != null && value.length() > 0) {
+            StudioEncryption se = StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM);
+            // Set default encrypt and decrypt methods
+            if (this.encrypt == null) {
+                this.encrypt = (src) -> se.encrypt(src);
+            }
+            if (this.decrypt == null) {
+                this.decrypt = (src) -> {
+                    if (src != null && StudioEncryption.hasEncryptionSymbol(src)) {
+                        return se.decrypt(src);
+                    }
+                    return src;
+                };
+            }
             String newValue = null;
             if (encrypt) {
-                newValue = CryptoHelper.getDefault().encrypt(value);
+                newValue = this.encrypt.apply(value);
             } else {
-                newValue = CryptoHelper.getDefault().decrypt(value);
+                newValue = this.decrypt.apply(value);
             }
             if (newValue != null) { // if enable to encrypt/decrypt will return the new value.
                 return newValue;
@@ -1208,4 +1226,14 @@ public class ConnectionImpl extends AbstractMetadataObjectImpl implements Connec
         }
         return value;
     }
+
+    /**
+     * @generated NOT
+     */
+    public void setEncryptAndDecryptFuncPair(java.util.function.Function<String, String> encrypt,
+            java.util.function.Function<String, String> decrypt) {
+        this.encrypt = encrypt;
+        this.decrypt = decrypt;
+
+    }
 } // ConnectionImpl

main/plugins/org.talend.model/src/main/java/org/talend/cwm/helper/ConnectionHelper.java

@@ -20,7 +20,6 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.Logger;
 import org.apache.log4j.Priority;
 import org.eclipse.emf.common.util.EList;
 import org.eclipse.emf.ecore.EObject;
@@ -43,7 +42,8 @@ import org.talend.cwm.relational.TdColumn;
 import org.talend.cwm.softwaredeployment.TdSoftwareSystem;
 import org.talend.cwm.xml.TdXmlElementType;
 import org.talend.cwm.xml.TdXmlSchema;
-import org.talend.daikon.security.CryptoHelper;
+import org.talend.utils.security.CryptoMigrationUtil;
+import org.talend.utils.security.StudioEncryption;
 
 import orgomg.cwm.foundation.softwaredeployment.Component;
 import orgomg.cwm.foundation.softwaredeployment.DataManager;
@@ -68,14 +68,6 @@ public class ConnectionHelper {
 
     public static final String DOT_STRING = "."; //$NON-NLS-1$
 
-    // MOD xqliu 2011-07-04 feature 22201
-    // public static final String PASSPHRASE = "99ZwBDt1L9yMX2ApJx fnv94o99OeHbCGuIHTy22
-    // V9O6cZ2i374fVjdV76VX9g49DG1r3n90hT5c1"; //$NON-NLS-1$
-
-    // ~
-
-    private static Logger log = Logger.getLogger(ConnectionHelper.class);
-
     /**
      * Method "createTdDataProvider" creates a data provider with the given name.
      *
@@ -1111,8 +1103,8 @@ public class ConnectionHelper {
                 boolean cleanFromNewWay = false;
                 String originalValue = tempValue;
                 try {
-                    tempValue = getDecryptPassword(originalValue);
-                    String encryptFromTempValue = getEncryptPassword(tempValue);
+                    tempValue = CryptoMigrationUtil.decrypt(originalValue);
+                    String encryptFromTempValue = CryptoMigrationUtil.encrypt(tempValue);
                     if (!StringUtils.equals(originalValue, encryptFromTempValue)) {
                         cleanFromNewWay = true;
                     }
@@ -1164,7 +1156,7 @@ public class ConnectionHelper {
      * @return
      */
     public static String getDecryptPassword(String password) {
-        return CryptoHelper.getDefault().decrypt(password);
+        return StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM).decrypt(password);
     }
 
     /**
@@ -1174,7 +1166,7 @@ public class ConnectionHelper {
      * @return
      */
     public static String getEncryptPassword(String password) {
-        return CryptoHelper.getDefault().encrypt(password);
+        return StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM).encrypt(password);
     }
 
     /**

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/impl/ContextParameterTypeImpl.java

@@ -10,9 +10,9 @@ import org.eclipse.emf.ecore.EClass;
 import org.eclipse.emf.ecore.impl.ENotificationImpl;
 import org.eclipse.emf.ecore.impl.EObjectImpl;
 import org.talend.commons.utils.PasswordEncryptUtil;
-import org.talend.daikon.security.CryptoHelper;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.TalendFilePackage;
+import org.talend.utils.security.StudioEncryption;
 
 /**
  * <!-- begin-user-doc --> An implementation of the model object '<em><b>Context Parameter Type</b></em>'. <!--
@@ -345,7 +345,7 @@ public class ContextParameterTypeImpl extends EObjectImpl implements ContextPara
 
     public String getRawValue() {
         if (value != null && value.length() > 0 && PasswordEncryptUtil.isPasswordType(getType())) {
-            String decryptValue = CryptoHelper.getDefault().decrypt(value);
+            String decryptValue = StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM).decrypt(value);
             if (decryptValue != null) {
                 return decryptValue;
             }
@@ -366,7 +366,8 @@ public class ContextParameterTypeImpl extends EObjectImpl implements ContextPara
 
     public void setRawValue(String newValue) {
         if (newValue != null && newValue.length() > 0 && PasswordEncryptUtil.isPasswordType(getType())) {
-            String encryptValue = CryptoHelper.getDefault().encrypt(newValue);
+            String encryptValue = StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM)
+                    .encrypt(newValue);
             if (encryptValue != null) {
                 setValue(encryptValue);
                 return;

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/impl/ElementParameterTypeImpl.java

@@ -17,10 +17,11 @@ import org.eclipse.emf.ecore.impl.EObjectImpl;
 import org.eclipse.emf.ecore.util.EObjectContainmentEList;
 import org.eclipse.emf.ecore.util.InternalEList;
 import org.talend.commons.utils.PasswordEncryptUtil;
-import org.talend.daikon.security.CryptoHelper;
 import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.ElementValueType;
 import org.talend.designer.core.model.utils.emf.talendfile.TalendFilePackage;
+import org.talend.utils.security.CryptoMigrationUtil;
+import org.talend.utils.security.StudioEncryption;
 
 /**
  * <!-- begin-user-doc --> An implementation of the model object '<em><b>Element Parameter Type</b></em>'. <!--
@@ -297,7 +298,14 @@ public class ElementParameterTypeImpl extends EObjectImpl implements ElementPara
 
     public String getRawValue() {
         if (value != null && value.length() > 0 && PasswordEncryptUtil.isPasswordField(getField())) {
-            String decrypt = CryptoHelper.getDefault().decrypt(value);
+            String decrypt = null;
+            if (StudioEncryption.hasEncryptionSymbol(value)) {
+                decrypt = StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM).decrypt(value);
+            } else {
+                // Some migration task: GenerateJobPomMigrationTask invokes this method
+                decrypt = CryptoMigrationUtil.decrypt(value);
+            }
+
             if (decrypt != null) {
                 return decrypt;
             }
@@ -318,7 +326,8 @@ public class ElementParameterTypeImpl extends EObjectImpl implements ElementPara
 
     public void setRawValue(String newValue) {
         if (newValue != null && newValue.length() > 0 && PasswordEncryptUtil.isPasswordField(getField())) {
-            String encryptValue = CryptoHelper.getDefault().encrypt(newValue);
+            String encryptValue = StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM)
+                    .encrypt(newValue);
             if (encryptValue != null) {
                 setValue(encryptValue);
                 return;

main/plugins/org.talend.model/src/main/java/org/talend/designer/core/model/utils/emf/talendfile/impl/ElementValueTypeImpl.java

@@ -9,9 +9,9 @@ import org.eclipse.emf.common.notify.Notification;
 import org.eclipse.emf.ecore.EClass;
 import org.eclipse.emf.ecore.impl.ENotificationImpl;
 import org.eclipse.emf.ecore.impl.EObjectImpl;
-import org.talend.daikon.security.CryptoHelper;
 import org.talend.designer.core.model.utils.emf.talendfile.ElementValueType;
 import org.talend.designer.core.model.utils.emf.talendfile.TalendFilePackage;
+import org.talend.utils.security.StudioEncryption;
 
 /**
  * <!-- begin-user-doc --> An implementation of the model object '<em><b>Element Value Type</b></em>'. <!-- end-user-doc
@@ -155,7 +155,7 @@ public class ElementValueTypeImpl extends EObjectImpl implements ElementValueTyp
 
     public String getRawValue() {
         if (value != null && value.length() > 0) {
-            String decrypt = CryptoHelper.getDefault().decrypt(value);
+            String decrypt = StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM).decrypt(value);
             if (decrypt != null) {
                 return decrypt;
             }
@@ -176,7 +176,7 @@ public class ElementValueTypeImpl extends EObjectImpl implements ElementValueTyp
 
     public void setValue(String value, boolean encrypt) {
         if (encrypt && value != null && value.length() > 0) {
-            String encryptValue = CryptoHelper.getDefault().encrypt(value);
+            String encryptValue = StudioEncryption.getStudioEncryption(StudioEncryption.EncryptionKeyName.SYSTEM).encrypt(value);
             if (encryptValue != null) {
                 setValue(encryptValue);
                 return;

main/plugins/org.talend.platform.logging/src/log4j2.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration>  
+    <Appenders>  
+        <Console name="Console" target="SYSTEM_OUT">  
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" />  
+        </Console>  
+    </Appenders>  
+    
+    <Loggers>  
+        <Root level="info">  
+            <AppenderRef ref="Console" />  
+        </Root>  
+    </Loggers>  
+</Configuration>

main/plugins/org.talend.rcp/src/main/java/org/talend/rcp/intro/Application.java

@@ -69,6 +69,7 @@ import org.talend.registration.wizards.license.LicenseWizardDialog;
 import org.talend.repository.ProjectManager;
 import org.talend.repository.model.IRepositoryService;
 import org.talend.repository.ui.login.LoginHelper;
+import org.talend.utils.StudioKeysFileCheck;
 
 /**
  * This class controls all aspects of the application's execution.
@@ -100,6 +101,8 @@ public class Application implements IApplication {
         }
         System.setProperty(TalendPropertiesUtil.PROD_APP, this.getClass().getName());
 
+        StudioKeysFileCheck.check(ConfigurationScope.INSTANCE.getLocation().toFile());
+        
         Display display = PlatformUI.createDisplay();
         try {
             // TUP-5816 don't put any code ahead of this part unless you make sure it won't trigger workspace
@@ -120,7 +123,7 @@ public class Application implements IApplication {
                 EclipseCommandLine.updateOrCreateExitDataPropertyWithCommand(EclipseCommandLine.CLEAN, null, false);
                 return IApplication.EXIT_RELAUNCH;
             }
-
+            
             StudioSSLContextProvider.setSSLSystemProperty();
             HttpProxyUtil.initializeHttpProxy();
             TalendProxySelector.getInstance();

main/plugins/org.talend.repository.items.importexport.ui/src/main/java/org/talend/repository/items/importexport/ui/messages.properties

@@ -24,6 +24,9 @@ ImportItemsWizardPage_sameIdProblemMessage=The 2 following items haves the same
 ImportItemsWizardPage_TarImport_badFormat = Source file is not a valid tar file.
 ImportItemsWizardPage_ZipImport_badFormat = Source file is not a valid Zip file.
 ImportItemsWizardPage_ErrorsMessage=The item '{0}' with different version {1} exists. Check the Recycle bin and empty it if needed.
+ImportItemsWizardPage_internalIdGroup=Internal id
+ImportItemsWizardPage_internalIdGroup_alwaysRegenId=Always regenerate id when import
+ImportItemsWizardPage_internalIdGroup_keepOrigId=Keep original internal id
 
 ImportItemsAction_title=Import items
 

main/plugins/org.talend.repository.items.importexport.ui/src/main/java/org/talend/repository/items/importexport/ui/wizard/imports/ImportItemsWizardPage.java

@@ -50,6 +50,9 @@ import org.eclipse.swt.events.SelectionAdapter;
 import org.eclipse.swt.events.SelectionEvent;
 import org.eclipse.swt.events.TraverseEvent;
 import org.eclipse.swt.events.TraverseListener;
+import org.eclipse.swt.layout.FormAttachment;
+import org.eclipse.swt.layout.FormData;
+import org.eclipse.swt.layout.FormLayout;
 import org.eclipse.swt.layout.GridData;
 import org.eclipse.swt.layout.GridLayout;
 import org.eclipse.swt.widgets.Button;
@@ -57,6 +60,7 @@ import org.eclipse.swt.widgets.Composite;
 import org.eclipse.swt.widgets.DirectoryDialog;
 import org.eclipse.swt.widgets.Display;
 import org.eclipse.swt.widgets.FileDialog;
+import org.eclipse.swt.widgets.Group;
 import org.eclipse.swt.widgets.Label;
 import org.eclipse.swt.widgets.Text;
 import org.eclipse.swt.widgets.TreeItem;
@@ -141,6 +145,8 @@ public class ImportItemsWizardPage extends WizardPage {
 
     private final ImportExportHandlersManager importManager = new ImportExportHandlersManager();
 
+    private Button regenIdBtn;
+
     /**
      *
      * DOC ggu ImportItemsWizardPage constructor comment.
@@ -498,8 +504,29 @@ public class ImportItemsWizardPage extends WizardPage {
      * @param workArea
      */
     private void createAdditionArea(Composite workArea) {
+        Composite optionsArea = new Composite(workArea, SWT.NONE);
+        FormLayout optAreaLayout = new FormLayout();
+        optionsArea.setLayout(optAreaLayout);
+        GridData gridData = new GridData(GridData.GRAB_HORIZONTAL | GridData.GRAB_VERTICAL | GridData.FILL_BOTH);
+        optionsArea.setLayoutData(gridData);
+
+        Group internalIdGroup = new Group(optionsArea, SWT.NONE);
+        internalIdGroup.setText(Messages.getString("ImportItemsWizardPage_internalIdGroup"));
+        internalIdGroup.setLayout(new GridLayout(1, true));
+        FormData internalIdGroupLayoutData = new FormData();
+        internalIdGroupLayoutData.top = new FormAttachment(0);
+        internalIdGroupLayoutData.left = new FormAttachment(0);
+        internalIdGroup.setLayoutData(internalIdGroupLayoutData);
+
+        regenIdBtn = new Button(internalIdGroup, SWT.RADIO);
+        regenIdBtn.setText(Messages.getString("ImportItemsWizardPage_internalIdGroup_alwaysRegenId"));
+
+        Button keepOrigIdBtn = new Button(internalIdGroup, SWT.RADIO);
+        keepOrigIdBtn.setText(Messages.getString("ImportItemsWizardPage_internalIdGroup_keepOrigId"));
+        keepOrigIdBtn.setSelection(true);
+
         // see feature 3949
-        this.overwriteButton = new Button(workArea, SWT.CHECK);
+        this.overwriteButton = new Button(optionsArea, SWT.CHECK);
         this.overwriteButton.setText(Messages.getString("ImportItemsWizardPage_overwriteItemsText")); //$NON-NLS-1$
         this.overwriteButton.addSelectionListener(new SelectionAdapter() {
 
@@ -511,6 +538,12 @@ public class ImportItemsWizardPage extends WizardPage {
             }
 
         });
+        FormData overwriteLayoutData = new FormData();
+        overwriteLayoutData.top = new FormAttachment(internalIdGroup, 5, SWT.BOTTOM);
+        overwriteLayoutData.left = new FormAttachment(internalIdGroup, 0, SWT.LEFT);
+        this.overwriteButton.setLayoutData(overwriteLayoutData);
+
+        internalIdGroup.setVisible(false);
     }
 
     protected boolean isEnableForExchange() {
@@ -982,6 +1015,7 @@ public class ImportItemsWizardPage extends WizardPage {
         }
 
         final boolean overwrite = overwriteButton.getSelection();
+        final boolean alwaysRegenId = regenIdBtn.getSelection();
         try {
             IRunnableWithProgress iRunnableWithProgress = new IRunnableWithProgress() {
 
@@ -1008,7 +1042,7 @@ public class ImportItemsWizardPage extends WizardPage {
                         EmfResourcesFactoryReader.INSTANCE.addOption(importOption, false);
 
                         importManager.importItemRecords(monitor, resManager, checkedItemRecords, overwrite,
-                                nodesBuilder.getAllImportItemRecords(), destinationPath);
+                                nodesBuilder.getAllImportItemRecords(), destinationPath, alwaysRegenId);
                     } finally {
                         EmfResourcesFactoryReader.INSTANCE.removOption(importOption, false);
                     }

main/plugins/org.talend.repository.items.importexport/src/main/java/org/talend/repository/items/importexport/handlers/HandlerUtil.java

@@ -165,6 +165,13 @@ public final class HandlerUtil {
                 public InputStream createInputStream(URI uri, Map<?, ?> options) throws IOException {
                     InputStream inputStream = null;
                     EPackage ePackage = resourceSet.getPackageRegistry().getEPackage(uri.toString());
+                    if (ePackage == null && uri.path().contains("CamelProperties")) { //$NON-NLS-1$
+                        // org.eclipse.emf.ecore.xmi.impl.XMLHandler.getPackageForURI(String)
+                        // after createInputStream still create resource and load while ePackage is null
+                        // throw IOexception skip process, XMLHandler.getPackageForURI will judge EPackage
+                        // then to throw PackageNotFoundException
+                        throw new IOException();
+                    }
                     if (ePackage != null || !"http".equals(uri.scheme())) { //$NON-NLS-1$
                         inputStream = super.createInputStream(uri, options);
                     } else {

main/plugins/org.talend.repository.items.importexport/src/main/java/org/talend/repository/items/importexport/handlers/ImportExportHandlersManager.java

@@ -417,6 +417,16 @@ public class ImportExportHandlersManager {
     public void importItemRecords(final IProgressMonitor progressMonitor, final ResourcesManager resManager,
             final List<ImportItem> checkedItemRecords, final boolean overwrite, final ImportItem[] allImportItemRecords,
             final IPath destinationPath) throws InvocationTargetException {
+        importItemRecords(progressMonitor, resManager, checkedItemRecords, overwrite, allImportItemRecords, destinationPath,
+                /*
+                 * disable by default, but provide possibility to enable it
+                 */
+                Boolean.getBoolean("studio.import.option.alwaysRegenId"));
+    }
+
+    public void importItemRecords(final IProgressMonitor progressMonitor, final ResourcesManager resManager,
+            final List<ImportItem> checkedItemRecords, final boolean overwrite, final ImportItem[] allImportItemRecords,
+            final IPath destinationPath, final boolean alwaysRegenId) throws InvocationTargetException {
         TimeMeasure.display = CommonsPlugin.isDebugMode();
         TimeMeasure.displaySteps = CommonsPlugin.isDebugMode();
         TimeMeasure.measureActive = CommonsPlugin.isDebugMode();
@@ -536,15 +546,31 @@ public class ImportExportHandlersManager {
                                     return;
                                 }
                                 if (itemRecord.isValid()) {
-                                    if (itemRecord.getState() == State.ID_EXISTED
-                                            || itemRecord.getState() == State.NAME_AND_ID_EXISTED_BOTH) {
+                                    if (alwaysRegenId || itemRecord.getState() == State.ID_EXISTED
+                                            || itemRecord.getState() == State.NAME_AND_ID_EXISTED_BOTH
+                                            || itemRecord.getState() == State.NAME_EXISTED) {
                                         String id = nameToIdMap.get(itemRecord.getProperty().getLabel()
                                                 + ERepositoryObjectType.getItemType(itemRecord.getProperty().getItem())
                                                         .toString());
                                         if (id == null) {
                                             try {
-                                                if (overwrite && itemRecord.getState() == State.NAME_AND_ID_EXISTED_BOTH) {
+                                                boolean reuseExistingId = false;
+                                                if (overwrite && (itemRecord.getState() == State.NAME_AND_ID_EXISTED_BOTH
+                                                        || itemRecord.getState() == State.NAME_EXISTED)) {
                                                     // just try to reuse the id of the item which will be overwrited
+                                                    reuseExistingId = true;
+                                                } else if (alwaysRegenId) {
+                                                    switch (itemRecord.getState()) {
+                                                    case NAME_EXISTED:
+                                                    case NAME_AND_ID_EXISTED:
+                                                    case NAME_AND_ID_EXISTED_BOTH:
+                                                        reuseExistingId = true;
+                                                        break;
+                                                    default:
+                                                        break;
+                                                    }
+                                                }
+                                                if (reuseExistingId) {
                                                     IRepositoryViewObject object = itemRecord.getExistingItemWithSameName();
                                                     if (object != null) {
                                                         if (ProjectManager.getInstance().isInCurrentMainProject(

main/plugins/org.talend.repository.items.importexport/src/main/java/org/talend/repository/items/importexport/manager/ChangeIdManager.java

@@ -24,6 +24,7 @@ import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Priority;
+import org.eclipse.emf.common.util.EList;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.model.metadata.builder.connection.Connection;
@@ -123,68 +124,101 @@ public class ChangeIdManager {
 
     public void changeIds() throws Exception {
         buildRefIds2ItemIdsMap();
-        for (Map.Entry<String, String> entry : oldId2NewIdMap.entrySet()) {
-            changeId(entry.getValue(), entry.getKey());
-        }
-    }
 
-    private void changeId(String newId, String oldId) throws Exception {
+        Map<String, Set<String>> changeIdMap = buildChangeIdMap();
 
-        if (newId == null || StringUtils.equals(newId, oldId)) {
-            return;
-        }
-
-        Set<String> relationIds = new HashSet<String>();
-        Collection<String> itemIds = refIds2ItemIdsMap.get(oldId);
-        if (itemIds != null && !itemIds.isEmpty()) {
-            relationIds.addAll(itemIds);
-        }
-
-        List<Relation> relations = getRelations(oldId);
-        for (Relation relation : relations) {
-            String projectLabel = ProcessUtils.getProjectLabelFromItemId(relation.getId());
-            if (projectLabel != null
-                    && !projectLabel.equals(ProjectManager.getInstance().getCurrentProject().getTechnicalLabel())) {
-                continue;
-            }
-            relationIds.add(ProcessUtils.getPureItemId(relation.getId()));
-        }
-
-        if (relationIds.isEmpty()) {
-            return;
-        }
-
-        Set<String> changedIds = new HashSet<String>();
-        for (String relationId : relationIds) {
-            String id = relationId;
-            if (!oldId2NewIdMap.containsKey(id)) {
+        for (Map.Entry<String, Set<String>> entry : changeIdMap.entrySet()) {
+            String oldEffectedId = entry.getKey();
+            if (!oldId2NewIdMap.containsKey(oldEffectedId)) {
                 // means didn't import this item
                 continue;
             }
-
-            if (changedIds.contains(id)) {
+            String newEffectedId = oldId2NewIdMap.get(oldEffectedId);
+            if (StringUtils.equals(oldEffectedId, newEffectedId)) {
                 continue;
-            } else {
-                changedIds.add(id);
             }
-
-            List<ImportItem> importItems = id2ImportItemsMap.get(id);
-            ERepositoryObjectType repType = importItems.get(0).getRepositoryType();
-            String newRelatedId = oldId2NewIdMap.get(id);
-            if (newRelatedId == null) {
+            if (StringUtils.isBlank(newEffectedId)) {
                 // means the id didn't be changed
-                newRelatedId = id;
+                newEffectedId = oldEffectedId;
             }
-            List<IRepositoryViewObject> repViewObjs = getAllVersion(newRelatedId, repType);
+            List<ImportItem> importItems = id2ImportItemsMap.get(oldEffectedId);
+            ERepositoryObjectType repType = importItems.get(0).getRepositoryType();
+            List<IRepositoryViewObject> repViewObjs = getAllVersion(newEffectedId, repType);
             if (repViewObjs != null && !repViewObjs.isEmpty()) {
                 for (IRepositoryViewObject repViewObj : repViewObjs) {
+                    Map<String, String> old2NewMap = new HashMap<>();
+                    for (String oldId : entry.getValue()) {
+                        String newId = oldId2NewIdMap.get(oldId);
+                        if (StringUtils.equals(newId, oldId)) {
+                            continue;
+                        }
+                        old2NewMap.put(oldId, newId);
+                    }
                     Property property = repViewObj.getProperty();
-                    changeRelated(newId, oldId, property, getCurrentProject());
+                    changeRelated(old2NewMap, property, getCurrentProject());
+                    String version = property.getVersion();
+                    for (ImportItem importItem : importItems) {
+                        if (StringUtils.equals(version, importItem.getItemVersion())) {
+                            // update property, it will be used in following steps
+                            importItem.setProperty(property);
+                            break;
+                        }
+                    }
                 }
             }
         }
     }
 
+    private Map<String, Set<String>> buildChangeIdMap() {
+        Map<String, Set<String>> effectedIdsMap = new HashMap<>();
+
+        for (Map.Entry<String, String> entry : oldId2NewIdMap.entrySet()) {
+            String oldId = entry.getKey();
+            String newId = entry.getValue();
+            if (newId == null || StringUtils.equals(newId, oldId)) {
+                continue;
+            }
+
+            Set<String> relationIds = new HashSet<String>();
+            Collection<String> itemIds = refIds2ItemIdsMap.get(oldId);
+            if (itemIds != null && !itemIds.isEmpty()) {
+                relationIds.addAll(itemIds);
+            }
+
+            List<Relation> relations = getRelations(oldId);
+            for (Relation relation : relations) {
+                relationIds.add(ProcessUtils.getPureItemId(relation.getId()));
+            }
+
+            if (relationIds.isEmpty()) {
+                continue;
+            }
+
+            Set<String> idSet = effectedIdsMap.get(oldId);
+            if (idSet == null) {
+                idSet = new HashSet<>();
+                effectedIdsMap.put(oldId, idSet);
+            }
+            idSet.addAll(relationIds);
+        }
+
+        Map<String, Set<String>> changeIdMap = new HashMap<>();
+        for (Map.Entry<String, Set<String>> entry : effectedIdsMap.entrySet()) {
+            String oldId = entry.getKey();
+            Set<String> effectedIds = entry.getValue();
+
+            for (String effectedId : effectedIds) {
+                Set<String> changeSet = changeIdMap.get(effectedId);
+                if (changeSet == null) {
+                    changeSet = new HashSet<>();
+                    changeIdMap.put(effectedId, changeSet);
+                }
+                changeSet.add(oldId);
+            }
+        }
+        return changeIdMap;
+    }
+
     private List<IRepositoryViewObject> getAllVersion(String id, ERepositoryObjectType repType) throws Exception {
         List<IRepositoryViewObject> repViewObjs = null;
         if (repType != null) {
@@ -253,16 +287,16 @@ public class ChangeIdManager {
         return relations;
     }
 
-    private void changeRelated(String newId, String oldId, Property property, org.talend.core.model.general.Project project)
-            throws Exception {
+    private void changeRelated(Map<String, String> old2NewMap, Property property,
+            org.talend.core.model.general.Project project) throws Exception {
         Item item = property.getItem();
         boolean modified = false;
         if (item instanceof ProcessItem) {
-            modified = changeRelatedProcess(newId, oldId, item);
+            modified = changeRelatedProcess(old2NewMap, item);
         } else if (item instanceof JobletProcessItem) {
-            modified = changeRelatedProcess(newId, oldId, item);
+            modified = changeRelatedProcess(old2NewMap, item);
         } else if (item instanceof ConnectionItem) {
-            modified = changeRelatedConnection(newId, oldId, (ConnectionItem) item);
+            modified = changeRelatedConnection(old2NewMap, (ConnectionItem) item);
         } else {
             throw new Exception("Unsupported id change: id[" + property.getId() + "], name[" + property.getLabel() + "]"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
         }
@@ -272,22 +306,41 @@ public class ChangeIdManager {
         }
     }
 
-    private boolean changeRelatedConnection(String newId, String oldId, ConnectionItem item) throws Exception {
+    private boolean changeRelatedConnection(Map<String, String> old2NewMap, ConnectionItem item) throws Exception {
         boolean modified = false;
         Connection conn = item.getConnection();
         String ctxId = conn.getContextId();
-        if (StringUtils.equals(oldId, ctxId)) {
-            conn.setContextId(newId);
-            modified = true;
-        } else {
+        for (Map.Entry<String, String> entry : old2NewMap.entrySet()) {
+            if (StringUtils.equals(entry.getKey(), ctxId)) {
+                conn.setContextId(entry.getValue());
+                modified = true;
+            }
+        }
+        if (!modified) {
             throw new Exception("Unhandled case for import: " + item.toString()); //$NON-NLS-1$
         }
         return modified;
     }
 
-    private boolean changeRelatedProcess(String newId, String oldId, Item item) throws Exception {
+    private boolean changeRelatedProcess(Map<String, String> old2NewMap, Item item) throws Exception {
         boolean modified = false;
 
+        ProcessItem processItem = (ProcessItem) item;
+        ProcessType processType = processItem.getProcess();
+
+        /**
+         * change context repository id
+         */
+        if (processType != null) {
+            EList context = processType.getContext();
+            if (context != null) {
+                for (Map.Entry<String, String> entry : old2NewMap.entrySet()) {
+                    changeValue(context, entry.getKey(), entry.getValue());
+                }
+                modified = true;
+            }
+        }
+
         /**
          * designerCoreService must not be null
          */
@@ -310,14 +363,18 @@ public class ChangeIdManager {
         // }
         IContextManager contextManager = process.getContextManager();
         if (contextManager != null) {
-            changeValue(contextManager.getListContext(), oldId, newId);
+            for (Map.Entry<String, String> entry : old2NewMap.entrySet()) {
+                changeValue(contextManager.getListContext(), entry.getKey(), entry.getValue());
+            }
             modified = true;
         }
 
         /**
          * 2. change elementParameters
          */
-        changeValue(process.getElementParameters(), oldId, newId);
+        for (Map.Entry<String, String> entry : old2NewMap.entrySet()) {
+            changeValue(process.getElementParameters(), entry.getKey(), entry.getValue());
+        }
         modified = true;
 
         /**
@@ -329,7 +386,9 @@ public class ChangeIdManager {
             while (nodeIter.hasNext()) {
                 INode node = nodeIter.next();
                 if (node != null) {
-                    changeParamValueOfNode(node, oldId, newId);
+                    for (Map.Entry<String, String> entry : old2NewMap.entrySet()) {
+                        changeParamValueOfNode(node, entry.getKey(), entry.getValue());
+                    }
                 }
             }
             modified = true;
@@ -337,7 +396,7 @@ public class ChangeIdManager {
 
         if (modified) {
             if (process instanceof IProcess2) {
-                ProcessType processType = ((IProcess2) process).saveXmlFile();
+                processType = ((IProcess2) process).saveXmlFile();
                 if (item instanceof ProcessItem) {
                     ((ProcessItem) item).setProcess(processType);
                 } else if (item instanceof JobletProcessItem) {

main/plugins/org.talend.repository.localprovider/src/main/java/org/talend/repository/localprovider/model/LocalRepositoryFactory.java

@@ -3251,8 +3251,7 @@ public class LocalRepositoryFactory extends AbstractEMFRepositoryFactory impleme
     protected void updatePreferenceProjectVersion(Project project) {
         String oldProductVersion = project.getEmfProject().getProductVersion();
         if (StringUtils.isNotBlank(oldProductVersion)) {
-            oldProductVersion = StringUtils.substringAfter(oldProductVersion, "-"); //$NON-NLS-1$
-            String oldVersion = VersionUtils.getTalendVersion(oldProductVersion);
+            String oldVersion = VersionUtils.getTalendPureVersion(oldProductVersion);
             String currentVersion = VersionUtils.getTalendVersion();
             if (!currentVersion.equals(oldVersion)) {
                 ProjectPreferenceManager prefManager = new ProjectPreferenceManager(project, "org.talend.designer.maven", false); //$NON-NLS-1$

main/plugins/org.talend.repository.mdm/META-INF/MANIFEST.MF

@@ -25,7 +25,8 @@ Require-Bundle: org.eclipse.ui.workbench,
  org.talend.repository.view.di;resolution:=optional,
  javax.mail,
  org.talend.repository.items.importexport,
- org.apache.xerces
+ org.apache.xerces,
+ org.talend.repository
 Bundle-ActivationPolicy: lazy
 Bundle-Vendor: .Talend SA.
 Bundle-ClassPath: .

main/plugins/org.talend.repository.mdm/resources/UpdateReport.xsd

@@ -29,6 +29,7 @@
 			<xsd:element maxOccurs="1" minOccurs="0" name="DataModel" nillable="true" type="xsd:string"/>
 			<xsd:element maxOccurs="1" minOccurs="1" name="Concept" nillable="false" type="xsd:string"/>
 			<xsd:element maxOccurs="1" minOccurs="1" name="Key" nillable="false" type="xsd:string"/>
+			<xsd:element maxOccurs="1" minOccurs="0" name="PrimaryKeyInfo" nillable="true" type="xsd:string"/>
 			<xsd:element maxOccurs="unbounded" minOccurs="0" name="Item" nillable="false">
 				<xsd:complexType mixed="false">
 					<xsd:sequence maxOccurs="1" minOccurs="1">