fix(TDQ-18322) throw NPE when retrieve the schema on Sybase (#3234)

* fix(TUP-26596)Proxy of libraries not working when studio have no (#3165) (#3172)

* fix(TUP-26596)Proxy of libraries not working when studio have no
internet access

* fix(TUP-26596)Proxy of libraries not working when studio have no
internet access

* fix(TUP-26596)Proxy of libraries not working when studio have no
internet access

* fix(TUP-26596)Proxy of libraries not working when studio have no
internet access

* fix(TUP-26189): [7.2.1] snowflake metadata connection with proxy failure (#3101) (#3114)

* fix(TUP-26189): [7.2.1] snowflake metadata connection with proxy failure

https://jira.talendforge.org/browse/TUP-26189

* fix(TUP-26793)JDBC Redshift in context mode still asked for jars
https://jira.talendforge.org/browse/TUP-26793

* fix(TUP-26212): Share more than one car component to nexus3.19.1 will fail to generate the right index file (#3100) (#3120)

* fix(TUP-26212): Share more than one car component to nexus3.19.1 will
fail to generate the right index file

https://jira.talendforge.org/browse/TUP-26212

* fix(TUP-26212): Share more than one car component to nexus3.19.1 will
fail to generate the right index file

https://jira.talendforge.org/browse/TUP-26212

Co-authored-by: bhe-talendbj <52311292+bhe-talendbj@users.noreply.github.com>

* fix(TUP-26539): Unassociate data from associateListeners (#3169)

* fix(TUP-26539): Fix wrong tree node implmentation

* fix(TUP-26539): add test case

* fix(TUP-26539): Remove reference to itself

* fix(TUP-26539): Return root node

* fix(TUP-26539): Update test case

* fix(TUP-26539): update test case

* fix(TUP-26539): Unassociate unnecessary listeners

* fix(TUP-26539): Revert changes of RepositoryNode

* fix(TUP-26539): Add override annotation

* fix(TUP-26876):NoClassDefFoundError when I run spark job with JobServer

Co-authored-by: hzhao-talendbj <49395568+hzhao-talendbj@users.noreply.github.com>
Co-authored-by: kjwang <kjwang@talend.com>
Co-authored-by: zhangjie <jie.zhang@talend.com>
Co-authored-by: Chao MENG <cmeng@talend.com>
Co-authored-by: bhe-talendbj <52311292+bhe-talendbj@users.noreply.github.com>

main/plugins/org.talend.commons.runtime/META-INF/MANIFEST.MF

@@ -13,7 +13,8 @@ Require-Bundle: org.apache.log4j;visibility:=reexport,
  org.talend.libraries.apache,
  com.ibm.icu,
  org.apache.commons.io,
- org.talend.utils
+ org.talend.utils,
+ org.eclipse.core.net
 Export-Package: org.talend.commons,
  org.talend.commons.exception,
  org.talend.commons.i18n,

main/plugins/org.talend.commons.runtime/src/org/talend/commons/CommonsPlugin.java

@@ -17,6 +17,7 @@ import java.io.InputStream;
 import java.net.URL;
 
 import org.apache.commons.lang.ArrayUtils;
+import org.eclipse.core.net.proxy.IProxyService;
 import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.preferences.InstanceScope;
@@ -24,6 +25,7 @@ import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleActivator;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.ServiceReference;
+import org.osgi.util.tracker.ServiceTracker;
 import org.talend.commons.exception.ExceptionService;
 import org.talend.commons.runtime.debug.TalendDebugHandler;
 import org.talend.commons.utils.system.EclipseCommandLine;
@@ -55,6 +57,8 @@ public class CommonsPlugin implements BundleActivator {
     // TESB-17856: For commandline builds ESB Micorservice bundle
     private static boolean isESBMicorservice = false;
 
+    private static ServiceTracker proxyTracker;
+
     public static boolean isWorkbenchCreated() {
         return isWorkbenchCreated;
     }
@@ -179,4 +183,13 @@ public class CommonsPlugin implements BundleActivator {
         return FileLocator.toFileURL(entry);
     }
 
+    public static IProxyService getProxyService() {
+        if (proxyTracker == null) {
+            proxyTracker = new ServiceTracker(Platform.getBundle(PLUGIN_ID).getBundleContext(), IProxyService.class.getName(),
+                    null);
+            proxyTracker.open();
+        }
+        return (IProxyService) proxyTracker.getService();
+    }
+
 }

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/network/TalendProxySelector.java

@@ -13,6 +13,8 @@
 package org.talend.commons.utils.network;
 
 import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
 import java.net.InetSocketAddress;
 import java.net.PasswordAuthentication;
 import java.net.Proxy;
@@ -35,8 +37,14 @@ import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Priority;
+import org.eclipse.core.internal.net.ProxyManager;
+import org.eclipse.core.net.proxy.IProxyService;
+import org.talend.commons.CommonsPlugin;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.i18n.internal.Messages;
+import org.talend.daikon.sandbox.properties.ClassLoaderIsolatedSystemProperties;
+
+import sun.net.spi.DefaultProxySelector;
 
 /**
  * DOC cmeng  class global comment. Detailled comment
@@ -51,9 +59,23 @@ public class TalendProxySelector extends ProxySelector {
 
     private static final String PROP_ALLOW_PROXY_REDIRECT_EXCLUDE = "talend.studio.proxy.redirect.whiteList";
 
+    private static final String PROP_EXECUTE_CONNECTION_FAILED = "talend.studio.proxy.executeConnectionFailed";
+
+    private static final String PROP_UPDATE_SYSTEM_PROPERTIES_FOR_JRE = "talend.studio.proxy.jre.updateSystemProperties";
+
+    private static final String PROP_CHECK_PROXY = "talend.studio.proxy.checkProxy";
+
+    private static final String PROP_VALIDATE_URI = "talend.studio.proxy.validateUri";
+
+    private static final String PROP_PROXY_SELECTOR = "talend.studio.proxy.selector";
+
+    private static final String PROP_PROXY_SELECTOR_DEFAULT = "default";
+
+    private static final String PROP_PROXY_SELECTOR_JRE = "jre";
+
     private static final String PROP_PROXY_HOST_MAP = "talend.studio.proxy.hostMap";
 
-    private static final String PROP_DISABLE_DEFAULT_SELECTOR = "talend.studio.proxy.disableDefaultSelector";
+    private static final String PROP_DISABLE_DEFAULT_SELECTOR_PROVIDER = "talend.studio.proxy.disableDefaultSelectorProvider";
 
     /**
      * Example: update.talend.com,socket:http,https:http;nexus.talend.com,socket,http;,socket:http
@@ -67,7 +89,20 @@ public class TalendProxySelector extends ProxySelector {
 
     private static final String KEY_DEFAULT = ":default:";
 
-    private ProxySelector defaultSelector;
+    private static Field uriHostField;
+
+    private static Method proxyManagerUpdateSystemPropertiesFunc;
+
+    private static boolean checkProxy = Boolean.valueOf(System.getProperty(PROP_CHECK_PROXY, Boolean.TRUE.toString()));
+
+    /**
+     * Note: eclipse default selector may be different between TOS and TIS, TOS may use jre one, TIS may use egit one
+     */
+    private ProxySelector eclipseDefaultSelector;
+
+    private ProxySelector jreDefaultSelector;
+
+    private EProxySelector eProxySelector;
 
     final private Map<Object, Collection<IProxySelectorProvider>> selectorProviders;
 
@@ -83,15 +118,36 @@ public class TalendProxySelector extends ProxySelector {
 
     private boolean allowProxyRedirect = false;
 
-    private boolean disableDefaultSelector = false;
+    private boolean disableDefaultSelectorProvider = false;
 
-    private TalendProxySelector(final ProxySelector defaultSelector) {
-        this.defaultSelector = defaultSelector;
+    private boolean validateUri = true;
+
+    private boolean executeConnectionFailed = true;
+
+    private boolean updateSystemPropertiesForJre = true;
+
+    private TalendProxySelector(final ProxySelector eclipseDefaultSelector) {
+        this.eclipseDefaultSelector = eclipseDefaultSelector;
+        this.jreDefaultSelector = new DefaultProxySelector();
 
         selectorProviders = Collections.synchronizedMap(new HashMap<>());
         allowProxyRedirect = Boolean.valueOf(System.getProperty(PROP_ALLOW_PROXY_REDIRECT, Boolean.FALSE.toString()));
-        disableDefaultSelector = Boolean.valueOf(System.getProperty(PROP_DISABLE_DEFAULT_SELECTOR, Boolean.FALSE.toString()));
+        disableDefaultSelectorProvider = Boolean
+                .valueOf(System.getProperty(PROP_DISABLE_DEFAULT_SELECTOR_PROVIDER, Boolean.FALSE.toString()));
         printProxyLog = Boolean.valueOf(System.getProperty(PROP_PRINT_LOGS, Boolean.FALSE.toString()));
+        validateUri = Boolean.valueOf(System.getProperty(PROP_VALIDATE_URI, Boolean.TRUE.toString()));
+        executeConnectionFailed = Boolean.valueOf(System.getProperty(PROP_EXECUTE_CONNECTION_FAILED, Boolean.TRUE.toString()));
+        updateSystemPropertiesForJre = Boolean
+                .valueOf(System.getProperty(PROP_UPDATE_SYSTEM_PROPERTIES_FOR_JRE, Boolean.TRUE.toString()));
+
+        switch (System.getProperty(PROP_PROXY_SELECTOR, PROP_PROXY_SELECTOR_DEFAULT).toLowerCase()) {
+        case PROP_PROXY_SELECTOR_JRE:
+            this.eProxySelector = EProxySelector.jre;
+            break;
+        default:
+            this.eProxySelector = EProxySelector.eclipse_default;
+            break;
+        }
 
         initHostMap();
         initRedirectList();
@@ -176,6 +232,18 @@ public class TalendProxySelector extends ProxySelector {
             synchronized (instanceLock) {
                 if (instance == null) {
                     instance = new TalendProxySelector(proxySelector);
+                    try {
+                        uriHostField = URI.class.getDeclaredField("host");
+                        uriHostField.setAccessible(true);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
+                    try {
+                        proxyManagerUpdateSystemPropertiesFunc = ProxyManager.class.getDeclaredMethod("updateSystemProperties");
+                        proxyManagerUpdateSystemPropertiesFunc.setAccessible(true);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
                 }
             }
         }
@@ -188,34 +256,72 @@ public class TalendProxySelector extends ProxySelector {
                     return null;
                 }
             });
-            if (instance.getDefaultProxySelector() == null
+            if (instance.getEclipseDefaultSelector() == null
                     || (proxySelector != null && proxySelector.getClass().getName().endsWith(ECLIPSE_PROXY_SELECTOR))) {
-                instance.setDefaultProxySelector(proxySelector);
+                instance.setEclipseDefaultSelector(proxySelector);
             }
         }
 
         return instance;
     }
 
+    public static void checkProxy() {
+        if (!checkProxy) {
+            return;
+        }
+        try {
+            TalendProxySelector.getInstance();
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
     @Override
     public List<Proxy> select(final URI uri) {
+        if (printProxyLog) {
+            ExceptionHandler.log("TalendProxySelector.select " + uri);
+        }
+        if (uri == null) {
+            return Collections.EMPTY_LIST;
+        }
+        URI validatedUri = validateUri(uri);
         Set<Proxy> results = new LinkedHashSet<>();
 
         try {
-            final Set<Proxy> resultFromProviders = getProxysFromProviders(uri);
+            final Set<Proxy> resultFromProviders = getProxysFromProviders(validatedUri);
             if (resultFromProviders != null && !resultFromProviders.isEmpty()) {
                 results.addAll(resultFromProviders);
             }
         } catch (Exception e) {
             ExceptionHandler.process(e);
         }
+        if (printProxyLog) {
+            ExceptionHandler.log("TalendProxySelector.resultFromProviders " + results);
+        }
 
         ProxySelector defaultProxySelector = getDefaultProxySelector();
+        if (printProxyLog) {
+            ExceptionHandler.log("TalendProxySelector.defaultProxySelector " + defaultProxySelector);
+        }
         if (defaultProxySelector != null) {
-            URI newUri = getNewUri(uri);
-            List<Proxy> defaultProxys = defaultProxySelector.select(newUri);
+            /**
+             * don't validate uri here, so that we can know whether it is an issue uri
+             */
+            URI newUri = getNewUri(validatedUri, false);
+            List<Proxy> defaultProxys = null;
+            if (validateUri && StringUtils.isBlank(newUri.getHost())) {
+                /**
+                 * If host is blank, force to use jre proxy selector to avoid the eclipse proxy selector bug
+                 */
+                defaultProxys = getJreProxySelector().select(newUri);
+            } else {
+                defaultProxys = defaultProxySelector.select(newUri);
+            }
+            if (printProxyLog) {
+                ExceptionHandler.log("TalendProxySelector.defaultProxys " + defaultProxys);
+            }
             try {
-                results.addAll(filterProxys(uri, defaultProxys));
+                results.addAll(filterProxys(validatedUri, defaultProxys));
             } catch (Exception e) {
                 results.addAll(defaultProxys);
                 ExceptionHandler.process(e);
@@ -229,6 +335,54 @@ public class TalendProxySelector extends ProxySelector {
         return new LinkedList<Proxy>(results);
     }
 
+    private URI validateUri(URI uri) {
+        if (!validateUri) {
+            return uri;
+        }
+
+        URI validatedUri = null;
+        try {
+            /**
+             * DON'T use URI.create(), MUST use the conductor which requires authority
+             */
+            validatedUri = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), uri.getQuery(), uri.getFragment());
+
+            /**
+             * Validate the host, if the host is empty, it will cause the eclipse selector to return dirrect
+             */
+            if (StringUtils.isBlank(validatedUri.getHost())) {
+                String authority = validatedUri.getAuthority();
+                if (StringUtils.isNotBlank(authority)) {
+                    // example: https://u:p@www.company.com:8081/path/a?param=b
+                    String host = null;
+                    int userInfoIndex = authority.indexOf('@');
+                    if (0 <= userInfoIndex) {
+                        authority = authority.substring(userInfoIndex + 1);
+                    }
+                    int portIndex = authority.lastIndexOf(':');
+                    if (0 <= portIndex) {
+                        host = authority.substring(0, portIndex);
+                    }
+                    try {
+                        uriHostField.set(validatedUri, host);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
+                }
+
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+            if (validatedUri == null) {
+                validatedUri = uri;
+            }
+        }
+        if (printProxyLog) {
+            ExceptionHandler.log("After validate: " + uri + " -> " + validatedUri);
+        }
+        return validatedUri;
+    }
+
     private List<Proxy> filterProxys(final URI uri, List<Proxy> defaultProxys) {
         List<Proxy> result = new ArrayList<>();
         if (defaultProxys != null && !defaultProxys.isEmpty()) {
@@ -264,38 +418,48 @@ public class TalendProxySelector extends ProxySelector {
         return result;
     }
 
-    private URI getNewUri(URI uri) {
+    private URI getNewUri(URI uri, boolean validateUri) {
         URI newUri = uri;
         if (newUri != null) {
-            String host = newUri.getHost();
-            Map<String, String> protocolMap = null;
-            if (StringUtils.isNotBlank(host)) {
-                protocolMap = hostMap.get(host.toLowerCase());
-            }
-            if (protocolMap == null) {
-                protocolMap = hostMap.get(KEY_DEFAULT);
-            }
+            try {
+                // get host before new URI, because the host may be set manually due to URI issue
+                String host = newUri.getHost();
+                newUri = new URI(newUri.getScheme(), newUri.getAuthority(), newUri.getPath(), newUri.getQuery(),
+                        newUri.getFragment());
+                Map<String, String> protocolMap = null;
+                if (StringUtils.isNotBlank(host)) {
+                    protocolMap = hostMap.get(host.toLowerCase());
+                }
+                if (protocolMap == null) {
+                    protocolMap = hostMap.get(KEY_DEFAULT);
+                }
 
-            if (protocolMap != null) {
-                String schema = newUri.getScheme();
-                if (schema != null) {
-                    String lowercasedProtocol = schema.toLowerCase();
-                    String preferedProtocol = protocolMap.get(lowercasedProtocol);
-                    if (StringUtils.isNotBlank(preferedProtocol)) {
-                        try {
-                            newUri = new URI(preferedProtocol, newUri.getUserInfo(), newUri.getHost(), newUri.getPort(),
-                                    newUri.getPath(), newUri.getQuery(), newUri.getFragment());
-                        } catch (URISyntaxException e) {
-                            if (printProxyLog) {
-                                ExceptionHandler.process(new Exception(
-                                        Messages.getString("TalendProxySelector.exception.proxySelectionError", newUri), e),
-                                        Priority.WARN);
-                            }
+                if (protocolMap != null) {
+                    String schema = newUri.getScheme();
+                    if (schema != null) {
+                        String lowercasedProtocol = schema.toLowerCase();
+                        String preferedProtocol = protocolMap.get(lowercasedProtocol);
+                        if (StringUtils.isNotBlank(preferedProtocol)) {
+                            /**
+                             * Note: MUST use the constructor which requires authority, because some uri may be illegal,
+                             * then host info will be stored in authority field instead of host filed
+                             */
+                            newUri = new URI(preferedProtocol, newUri.getAuthority(), newUri.getPath(), newUri.getQuery(),
+                                    newUri.getFragment());
                         }
                     }
                 }
+            } catch (URISyntaxException e) {
+                if (printProxyLog) {
+                    ExceptionHandler.process(
+                            new Exception(Messages.getString("TalendProxySelector.exception.proxySelectionError", uri), e),
+                            Priority.WARN);
+                }
             }
         }
+        if (validateUri) {
+            newUri = validateUri(newUri);
+        }
         return newUri;
     }
 
@@ -395,11 +559,45 @@ public class TalendProxySelector extends ProxySelector {
     }
 
     public ProxySelector getDefaultProxySelector() {
-        return defaultSelector;
+        switch (eProxySelector) {
+        case jre:
+            return getJreProxySelector();
+        default:
+            return eclipseDefaultSelector;
+        }
     }
 
-    public void setDefaultProxySelector(final ProxySelector selector) {
-        defaultSelector = selector;
+    private ProxySelector getJreProxySelector() {
+        try {
+            /**
+             * for tcompv0, daikon may create an isolated system properties for it, so proxies may be ignored in the new
+             * system properties; here we try to call the method to add proxies into the isolated system properties
+             */
+            if (updateSystemPropertiesForJre && ClassLoaderIsolatedSystemProperties.getInstance()
+                    .isIsolated(Thread.currentThread().getContextClassLoader())) {
+                if (printProxyLog) {
+                    ExceptionHandler.log("Before update jre proxy system properties for the isolated classloader, http.proxyHost="
+                            + System.getProperty("http.proxyHost"));
+                }
+                IProxyService proxyService = CommonsPlugin.getProxyService();
+                proxyManagerUpdateSystemPropertiesFunc.invoke(proxyService);
+                if (printProxyLog) {
+                    ExceptionHandler.log("After updated jre proxy system properties for the isolated classloader, http.proxyHost="
+                            + System.getProperty("http.proxyHost"));
+                }
+            }
+        } catch (Throwable e) {
+            ExceptionHandler.process(e);
+        }
+        return this.jreDefaultSelector;
+    }
+
+    public ProxySelector getEclipseDefaultSelector() {
+        return eclipseDefaultSelector;
+    }
+
+    public void setEclipseDefaultSelector(ProxySelector eclipseDefaultSelector) {
+        this.eclipseDefaultSelector = eclipseDefaultSelector;
     }
 
     @Override
@@ -413,21 +611,26 @@ public class TalendProxySelector extends ProxySelector {
             });
         }
 
-        ProxySelector defaultProxySelector = getDefaultProxySelector();
-        if (defaultProxySelector != null) {
-            defaultProxySelector.connectFailed(uri, sa, ioe);
+        if (executeConnectionFailed) {
+            /**
+             * Just try to make the behavior of jre proxy selector same like eclipse proxy selector
+             */
+            ProxySelector defaultProxySelector = getDefaultProxySelector();
+            if (defaultProxySelector != null) {
+                defaultProxySelector.connectFailed(uri, sa, ioe);
+            }
         }
     }
 
     public IProxySelectorProvider createDefaultProxySelectorProvider() {
-        if (disableDefaultSelector) {
+        if (disableDefaultSelectorProvider) {
             return null;
         }
         return new DefaultProxySelectorProvider(Thread.currentThread());
     }
 
     public IProxySelectorProvider createDefaultProxySelectorProvider(String host) {
-        if (disableDefaultSelector) {
+        if (disableDefaultSelectorProvider) {
             return null;
         }
         return new DefaultProxySelectorProvider(host);
@@ -445,6 +648,11 @@ public class TalendProxySelector extends ProxySelector {
         return possibleKeys;
     }
 
+    private enum EProxySelector {
+        eclipse_default,
+        jre
+    }
+
     private class DefaultProxySelectorProvider extends AbstractProxySelectorProvider {
 
         private Thread currentThread = null;
@@ -475,7 +683,7 @@ public class TalendProxySelector extends ProxySelector {
 
         @Override
         public boolean canHandle(URI uri) {
-            if (disableDefaultSelector) {
+            if (disableDefaultSelectorProvider) {
                 return false;
             }
             if (currentThread != null && Thread.currentThread() == currentThread) {

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java

@@ -70,6 +70,7 @@ import org.talend.commons.runtime.service.ITaCoKitService;
 import org.talend.commons.ui.gmf.util.DisplayUtils;
 import org.talend.commons.ui.runtime.exception.MessageBoxExceptionHandler;
 import org.talend.commons.utils.data.container.RootContainer;
+import org.talend.commons.utils.network.TalendProxySelector;
 import org.talend.commons.utils.time.TimeMeasure;
 import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.AbstractDQModelService;
@@ -2034,6 +2035,11 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
 
             TimeMeasure.begin("logOnProject"); //$NON-NLS-1$
             try {
+                /**
+                 * init/check proxy selector, in case default proxy selector is not registed yet
+                 */
+                TalendProxySelector.checkProxy();
+
                 System.getProperties().put("ReadOnlyUser", Boolean.FALSE.toString()); //$NON-NLS-1$
 
                 // remove the auto-build to enhance the build speed and application's use
@@ -2409,9 +2415,14 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
     @Override
     @SuppressWarnings("unchecked")
     public void executeRepositoryWorkUnit(RepositoryWorkUnit workUnit) {
+        checkProxySettings();
         this.repositoryFactoryFromProvider.executeRepositoryWorkUnit(workUnit);
     }
 
+    private void checkProxySettings() {
+        TalendProxySelector.checkProxy();
+    }
+
     @Override
     public void unloadResources(Property property) throws PersistenceException {
         repositoryFactoryFromProvider.unloadResources(property);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/TalendLibsServerManager.java

@@ -187,11 +187,13 @@ public class TalendLibsServerManager {
      * Check user library connection with the setting from remote administrator
      * 
      * @return
+     * @throws PersistenceException
      */
-    public boolean canConnectUserLibrary() {
+    public boolean canConnectUserLibrary() throws PersistenceException {
         boolean canConnect = false;
         IProxyRepositoryFactory factory = CoreRuntimePlugin.getInstance().getProxyRepositoryFactory();
         RepositoryContext repositoryContext = factory.getRepositoryContext();
+        ArtifactRepositoryBean bean = null;
         try {
             if (repositoryContext != null && repositoryContext.getFields() != null && !factory.isLocalConnectionProvider()
                     && !repositoryContext.isOffline()) {
@@ -208,7 +210,7 @@ public class TalendLibsServerManager {
                         && GlobalServiceRegister.getDefault().isServiceRegistered(IRemoteService.class)) {
                     IRemoteService remoteService = (IRemoteService) GlobalServiceRegister.getDefault()
                             .getService(IRemoteService.class);
-                    ArtifactRepositoryBean bean = remoteService.getLibNexusServer(userName, password, adminUrl);
+                    bean = remoteService.getLibNexusServer(userName, password, adminUrl);
                     if (bean != null) {
                         IRepositoryArtifactHandler handler = RepositoryArtifactHandlerManager.getRepositoryHandler(bean);
                         if (handler.checkConnection()) {
@@ -223,6 +225,9 @@ public class TalendLibsServerManager {
         } catch (Exception e) {
             ExceptionHandler.process(e);
         }
+        if (bean == null) {
+            throw new PersistenceException(Messages.getString("TalendLibsServerManager.cannotGetUserLibraryServer"));
+        }
         return canConnect;
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/process/LastGenerationInfo.java

@@ -32,8 +32,12 @@ public class LastGenerationInfo {
 
     private HashMap<String, Set<ModuleNeeded>> modulesNeededWithSubjobPerJob;
 
+    private HashMap<String, Set<ModuleNeeded>> highPriorityModuleNeededPerJob;
+
     private HashMap<String, Set<ModuleNeeded>> highPriorityModuleNeeded;
 
+    private HashMap<String, Set<ModuleNeeded>> testcaseModuleNeeded;
+
     private HashMap<String, Set<String>> routinesNeededWithSubjobPerJob;
 
     private HashMap<String, Set<String>> contextPerJob;
@@ -52,7 +56,9 @@ public class LastGenerationInfo {
         modulesNeededPerJob = new HashMap<String, Set<ModuleNeeded>>();
         contextPerJob = new HashMap<String, Set<String>>();
         modulesNeededWithSubjobPerJob = new HashMap<String, Set<ModuleNeeded>>();
+        highPriorityModuleNeededPerJob = new HashMap<>();
         highPriorityModuleNeeded = new HashMap<>();
+        testcaseModuleNeeded = new HashMap<>();
         lastGeneratedjobs = new HashSet<JobInfo>();
         routinesNeededPerJob = new HashMap<String, Set<String>>();
         routinesNeededWithSubjobPerJob = new HashMap<String, Set<String>>();
@@ -235,6 +241,22 @@ public class LastGenerationInfo {
         return routinesNeededPerJob.get(key);
     }
 
+    public Set<ModuleNeeded> getHighPriorityModuleNeededPerJob(String jobId, String jobVersion) {
+        String key = getProcessKey(jobId, jobVersion);
+        if (!highPriorityModuleNeededPerJob.containsKey(key)) {
+            highPriorityModuleNeededPerJob.put(key, new LinkedHashSet<>());
+        }
+        return highPriorityModuleNeededPerJob.get(key);
+    }
+
+    public void setHighPriorityModuleNeededPerJob(String jobId, String jobVersion, Set<ModuleNeeded> moduleNeeded) {
+        String key = getProcessKey(jobId, jobVersion);
+        if (!highPriorityModuleNeededPerJob.containsKey(key)) {
+            highPriorityModuleNeededPerJob.put(key, new LinkedHashSet<>());
+        }
+        highPriorityModuleNeededPerJob.get(key).addAll(moduleNeeded);
+    }
+
     public Set<ModuleNeeded> getHighPriorityModuleNeeded(String jobId, String jobVersion) {
         String key = getProcessKey(jobId, jobVersion);
         if (!highPriorityModuleNeeded.containsKey(key)) {
@@ -251,7 +273,20 @@ public class LastGenerationInfo {
         highPriorityModuleNeeded.get(key).addAll(moduleNeeded);
     }
 
+    public Set<ModuleNeeded> getTestcaseModuleNeeded(String jobId, String jobVersion) {
+        String key = getProcessKey(jobId, jobVersion);
+        if (!testcaseModuleNeeded.containsKey(key)) {
+            testcaseModuleNeeded.put(key, new HashSet<>());
+        }
+        return testcaseModuleNeeded.get(key);
+    }
+
+    public void setTestcaseModuleNeeded(String jobId, String jobVersion, Set<ModuleNeeded> modulesNeeded) {
+        testcaseModuleNeeded.put(getProcessKey(jobId, jobVersion), new HashSet<>(modulesNeeded));
+    }
+
     public void clearHighPriorityModuleNeeded() {
+        highPriorityModuleNeededPerJob.clear();
         highPriorityModuleNeeded.clear();
     }
 
@@ -292,26 +327,19 @@ public class LastGenerationInfo {
         routinesNeededWithSubjobPerJob.put(key, new HashSet<String>(modulesNeeded));
     }
 
-    public void clearModulesNeededWithSubjobPerJob() {
-        if (!modulesNeededWithSubjobPerJob.isEmpty()) {
-            modulesNeededWithSubjobPerJob.clear();
-        }
-    }
-
-    /**
-     * Clear modules per job cache, not thread safe
-     */
-    public void clearModulesNeededPerJob() {
-        if (!modulesNeededPerJob.isEmpty()) {
-            modulesNeededPerJob.clear();
-        }
+    public void clearCaches() {
+        clearHighPriorityModuleNeeded();
+        modulesNeededPerJob.clear();
+        modulesNeededWithSubjobPerJob.clear();
+        testcaseModuleNeeded.clear();
     }
 
     public void clean() {
         modulesNeededPerJob.clear();
         routinesNeededPerJob.clear();
         modulesNeededWithSubjobPerJob.clear();
-        highPriorityModuleNeeded.clear();
+        clearHighPriorityModuleNeeded();
+        testcaseModuleNeeded.clear();
         routinesNeededWithSubjobPerJob.clear();
         contextPerJob.clear();
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/process/TalendProcessOptionConstants.java

@@ -64,7 +64,10 @@ public interface TalendProcessOptionConstants {
 
     public static final int MODULES_WITH_INDEPENDENT = 1 << 2;
 
-    // include joblet module but without joblet's dependencies
+    /**
+     * @Deprecated will get all modules of node inside joblet recursively, use {@link #MODULES_DEFAULT} instead.
+     */
+    @Deprecated
     public static final int MODULES_WITH_JOBLET = 1 << 3;
 
     public static final int MODULES_FOR_MR = 1 << 4;

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java

@@ -18,7 +18,6 @@ import java.util.Set;
 import org.apache.log4j.Level;
 import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.resources.IProject;
-import org.eclipse.core.resources.IResource;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
@@ -244,4 +243,6 @@ public interface IRunProcessService extends IService {
 
     public IFolder getJavaProjectExternalResourcesFolder(IProcess process);
 
+    public boolean isCIMode();
+
 }

main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java

@@ -565,8 +565,10 @@ public class ProcessorUtilities {
                     neededLibraries);
 
             // get all job testcases needed modules
-            neededLibraries.addAll(getAllJobTestcaseModules(selectedProcessItem));
-
+            Set<ModuleNeeded> testcaseModules = getAllJobTestcaseModules(selectedProcessItem);
+            LastGenerationInfo.getInstance().setTestcaseModuleNeeded(jobInfo.getJobId(), jobInfo.getJobVersion(),
+                    testcaseModules);
+            neededLibraries.addAll(testcaseModules);
             // must install the needed libraries before generate codes with poms.
             CorePlugin.getDefault().getRunProcessService().updateLibraries(neededLibraries, currentProcess,
                     retrievedJarsForCurrentBuild);
@@ -1246,7 +1248,10 @@ public class ProcessorUtilities {
                         neededLibraries);
 
                 // get all job testcases needed modules
-                neededLibraries.addAll(getAllJobTestcaseModules(selectedProcessItem));
+                Set<ModuleNeeded> testcaseModules = getAllJobTestcaseModules(selectedProcessItem);
+                LastGenerationInfo.getInstance().setTestcaseModuleNeeded(jobInfo.getJobId(), jobInfo.getJobVersion(),
+                        testcaseModules);
+                neededLibraries.addAll(testcaseModules);
 
                 // must install the needed libraries before generate codes with poms.
                 CorePlugin.getDefault().getRunProcessService().updateLibraries(neededLibraries, currentProcess,
@@ -2850,6 +2855,10 @@ public class ProcessorUtilities {
         return jarFilePath;
     }
 
+    public static boolean isCIMode() {
+        return isCIMode;
+    }
+
     public static void setCIMode(boolean isCIMode) {
         ProcessorUtilities.isCIMode = isCIMode;
     }

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/MavenLibraryResolverProvider.java

@@ -25,7 +25,7 @@ import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
 import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
 import org.apache.maven.wagon.Wagon;
 import org.apache.maven.wagon.providers.file.FileWagon;
-import org.apache.maven.wagon.providers.http.HttpWagon;
+import org.apache.maven.wagon.providers.http.LightweightHttpWagon;
 import org.apache.maven.wagon.providers.http.LightweightHttpWagonAuthenticator;
 import org.apache.maven.wagon.providers.http.LightweightHttpsWagon;
 import org.codehaus.plexus.DefaultPlexusContainer;
@@ -238,9 +238,13 @@ public class MavenLibraryResolverProvider {
 
         LightweightHttpsWagon https = new LightweightHttpsWagon();
         https.setAuthenticator(new LightweightHttpWagonAuthenticator());
+        https.setPreemptiveAuthentication(true);
+        LightweightHttpWagon http = new LightweightHttpWagon();
+        http.setAuthenticator(new LightweightHttpWagonAuthenticator());
+        http.setPreemptiveAuthentication(true);
 
         pc.addComponent(https, Wagon.class, "https");
-        pc.addComponent(new HttpWagon(), Wagon.class, "http");
+        pc.addComponent(http, Wagon.class, "http");
         pc.addComponent(new FileWagon(), Wagon.class, "file");
 
         WagonTransporterFactory tf = (WagonTransporterFactory) locator.getService(TransporterFactory.class);

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/ProcessorDependenciesManager.java

@@ -13,7 +13,7 @@
 package org.talend.designer.maven.tools;
 
 import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -24,16 +24,19 @@ import org.apache.maven.model.Dependency;
 import org.apache.maven.model.Exclusion;
 import org.apache.maven.model.Model;
 import org.eclipse.core.runtime.IProgressMonitor;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.ILibraryManagerService;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.process.IProcess2;
 import org.talend.core.model.properties.ProcessItem;
+import org.talend.core.model.properties.Property;
 import org.talend.core.runtime.process.LastGenerationInfo;
 import org.talend.core.runtime.process.TalendProcessOptionConstants;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.designer.maven.utils.PomUtil;
+import org.talend.designer.maven.utils.SortableDependency;
 import org.talend.designer.runprocess.IProcessor;
 import org.talend.designer.runprocess.ProcessorException;
 
@@ -45,54 +48,71 @@ public class ProcessorDependenciesManager {
 
     private final IProcessor processor;
 
+    private final Property property;
+
     public ProcessorDependenciesManager(IProcessor processor) {
         this.processor = processor;
+        property = processor.getProperty();
     }
 
     /**
      * Will add the dependencies to the maven model.
      */
+    @SuppressWarnings({ "rawtypes", "unchecked" })
     public boolean updateDependencies(IProgressMonitor progressMonitor, Model model) throws ProcessorException {
         try {
-            List neededDependencies = new ArrayList<Dependency>();
-            Set<ModuleNeeded> neededLibraries = getAllModuleNeededWithTestCase();
-            Set<String> uniquDependenciesSet = new HashSet<>();
-
-            for (ModuleNeeded module : neededLibraries) {
-                Dependency dependency = null;
-                // if (module.getDeployStatus() == ELibraryInstallStatus.DEPLOYED) {
-                // }
-                final String mavenUri = module.getMavenUri();
-                if (uniquDependenciesSet.contains(mavenUri)) {
-                    continue; // must be same GAV, avoid the different other attrs for modules
-                }
-                uniquDependenciesSet.add(mavenUri);
-                dependency = PomUtil.createModuleDependency(mavenUri);
-                if (dependency != null) {
-                    if (module.isExcludeDependencies()) {
-                        Exclusion exclusion = new Exclusion();
-                        exclusion.setGroupId("*"); //$NON-NLS-1$
-                        exclusion.setArtifactId("*"); //$NON-NLS-1$
-                        dependency.addExclusion(exclusion);
-                    }
-                    neededDependencies.add(dependency);
+            Set<ModuleNeeded> neededLibraries = new HashSet<>();
+            Set<String> uniqueDependencies = new HashSet<>();
+            Set<ModuleNeeded> jobModules = getProcessNeededModules();
+            Set<ModuleNeeded> testcaseModules = getTestcaseNeededModules(property);
+            neededLibraries.addAll(jobModules);
+            neededLibraries.addAll(testcaseModules);
+            if (!neededLibraries.isEmpty()) {
+                if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibraryManagerService.class)) {
+                    ILibraryManagerService repositoryBundleService = GlobalServiceRegister.getDefault()
+                            .getService(ILibraryManagerService.class);
+                    repositoryBundleService.installModules(neededLibraries, null);
                 }
             }
-
-            java.util.Collections.sort(neededDependencies);
+            List neededDependencies = new ArrayList<>();
+            neededDependencies.addAll(convertToDependency(jobModules, uniqueDependencies, false));
+            neededDependencies.addAll(convertToDependency(testcaseModules, uniqueDependencies, true));
+            Collections.sort(neededDependencies);
             boolean fresh = false;
-            if (processor.getProperty() != null && processor.getProperty().getItem() != null
-                    && processor.getProcess() instanceof IProcess2) {
+            if (property != null && property.getItem() != null && processor.getProcess() instanceof IProcess2) {
                 // is standard job.
                 fresh = true;
             }
             return updateDependencies(progressMonitor, model, neededDependencies, fresh);
-
         } catch (Exception e) {
             throw new ProcessorException(e);
         }
     }
 
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    private List convertToDependency(Set<ModuleNeeded> neededLibraries, Set<String> uniqueDependencies, boolean optional) {
+        List neededDependencies = new ArrayList<>();
+        for (ModuleNeeded module : neededLibraries) {
+            final String mavenUri = module.getMavenUri();
+            if (uniqueDependencies.contains(mavenUri)) {
+                continue; // must be same GAV, avoid the different other attrs for modules
+            }
+            uniqueDependencies.add(mavenUri);
+            Dependency dependency = PomUtil.createModuleDependency(mavenUri);
+            if (dependency != null) {
+                ((SortableDependency) dependency).setAssemblyOptional(optional);
+                if (module.isExcludeDependencies()) {
+                    Exclusion exclusion = new Exclusion();
+                    exclusion.setGroupId("*"); //$NON-NLS-1$
+                    exclusion.setArtifactId("*"); //$NON-NLS-1$
+                    dependency.addExclusion(exclusion);
+                }
+                neededDependencies.add(dependency);
+            }
+        }
+        return neededDependencies;
+    }
+
     /**
      *
      * DOC ggu Comment method "updateDependencies". add the job Needed Libraries for current model.
@@ -159,35 +179,35 @@ public class ProcessorDependenciesManager {
         return changed;
     }
 
-    private Set<ModuleNeeded> getAllModuleNeededWithTestCase() throws PersistenceException {
-        // add the job modules.
-        Set<ModuleNeeded> neededLibraries = new HashSet<>();
-        Collection<ModuleNeeded> modulesNeeded = LastGenerationInfo.getInstance()
+    private Set<ModuleNeeded> getProcessNeededModules() {
+        Set<ModuleNeeded> neededLibraries = LastGenerationInfo.getInstance()
                 .getModulesNeededPerJob(processor.getProcess().getId(), processor.getProcess().getVersion());
-        if (modulesNeeded.isEmpty()) {
-            modulesNeeded = processor.getNeededModules(TalendProcessOptionConstants.MODULES_WITH_JOBLET);
-        }
-        neededLibraries.addAll(modulesNeeded);
-
-        // add testcase modules
-        if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
-            ITestContainerProviderService testcontainerService = (ITestContainerProviderService) GlobalServiceRegister
-                    .getDefault().getService(ITestContainerProviderService.class);
-            Set<ModuleNeeded> testcaseModules = null;
-            if (processor.getProperty() != null && processor.getProperty().getItem() instanceof ProcessItem) {
-                ProcessItem item = (ProcessItem) processor.getProperty().getItem();
-                testcaseModules = testcontainerService.getAllJobTestcaseModules(item);
-                neededLibraries.addAll(testcaseModules);
-                if (testcaseModules != null) {
-                    if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibraryManagerService.class)) {
-                        ILibraryManagerService repositoryBundleService = (ILibraryManagerService) GlobalServiceRegister
-                                .getDefault().getService(ILibraryManagerService.class);
-                        repositoryBundleService.installModules(neededLibraries, null);
-                    }
-                }
-            }
+        if (neededLibraries.isEmpty()) {
+            neededLibraries = processor.getNeededModules(TalendProcessOptionConstants.MODULES_DEFAULT);
         }
         return neededLibraries;
     }
 
+    public static Set<ModuleNeeded> getTestcaseNeededModules(Property property) {
+        if (property == null || !(property.getItem() instanceof ProcessItem)) {
+            return Collections.emptySet();
+        }
+        Set<ModuleNeeded> testcaseModules = LastGenerationInfo.getInstance().getTestcaseModuleNeeded(property.getId(),
+                property.getVersion());
+        if (testcaseModules.isEmpty()) {
+            if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
+                ITestContainerProviderService testcontainerService = GlobalServiceRegister.getDefault()
+                        .getService(ITestContainerProviderService.class);
+                try {
+                    testcaseModules = testcontainerService.getAllJobTestcaseModules((ProcessItem) property.getItem());
+                    LastGenerationInfo.getInstance().setTestcaseModuleNeeded(property.getId(), property.getVersion(),
+                            testcaseModules);
+                } catch (PersistenceException e) {
+                    ExceptionHandler.process(e);
+                }
+            }
+        }
+        return testcaseModules;
+    }
+
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenJobPom.java

@@ -18,18 +18,22 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.apache.maven.artifact.versioning.ComparableVersion;
 import org.apache.maven.model.Activation;
 import org.apache.maven.model.Dependency;
 import org.apache.maven.model.Model;
@@ -57,11 +61,10 @@ import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Project;
 import org.talend.core.model.properties.Property;
+import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.repository.utils.ItemResourceUtil;
-import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenConstants;
-import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.core.runtime.process.ITalendProcessJavaProject;
 import org.talend.core.runtime.process.JobInfoProperties;
 import org.talend.core.runtime.process.LastGenerationInfo;
@@ -75,6 +78,7 @@ import org.talend.core.utils.TemplateFileUtils;
 import org.talend.designer.maven.model.TalendMavenConstants;
 import org.talend.designer.maven.template.ETalendMavenVariables;
 import org.talend.designer.maven.template.MavenTemplateManager;
+import org.talend.designer.maven.tools.ProcessorDependenciesManager;
 import org.talend.designer.maven.utils.PomIdsHelper;
 import org.talend.designer.maven.utils.PomUtil;
 import org.talend.designer.maven.utils.SortableDependency;
@@ -620,139 +624,106 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
     }
 
     protected void updateDependencySet(IFile assemblyFile) {
-        Map<String, Dependency> jobCoordinateMap = new HashMap<String, Dependency>();
-        Set<JobInfo> childrenJobInfo = new HashSet<>();
-        if (!hasLoopDependency()) {
-            childrenJobInfo = getJobProcessor().getBuildChildrenJobs();
-        }
-
-        // add children jobs
-        for (JobInfo jobInfo : childrenJobInfo) {
-            Property property = jobInfo.getProcessItem().getProperty();
-            String coordinate =
-                    getCoordinate(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo),
-                            MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(property));
-            Dependency dependency = getDependencyObject(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo), PomIdsHelper.getJobVersion(property),
-                            MavenConstants.PACKAGING_JAR, null);
-            jobCoordinateMap.put(coordinate, dependency);
-        }
-
-        // add parent job
-        Property parentProperty = this.getJobProcessor().getProperty();
-        String parentCoordinate =
-                getCoordinate(PomIdsHelper.getJobGroupId(parentProperty), PomIdsHelper.getJobArtifactId(parentProperty),
-                        MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(parentProperty));
-        Dependency parentDependency = getDependencyObject(PomIdsHelper.getJobGroupId(parentProperty), PomIdsHelper.getJobArtifactId(parentProperty), PomIdsHelper.getJobVersion(parentProperty),
-                        MavenConstants.PACKAGING_JAR, null);
-        jobCoordinateMap.put(parentCoordinate, parentDependency);
-
-        // add talend libraries and codes
-        Map<String, Dependency> talendLibCoordinateMap = new HashMap<String, Dependency>();
-        String projectTechName = ProjectManager.getInstance().getProject(parentProperty).getTechnicalLabel();
-        String projectGroupId = PomIdsHelper.getProjectGroupId(projectTechName);
-
-        // codes
-        List<Dependency> dependencies = new ArrayList<>();
-        addCodesDependencies(dependencies);
-        for (Dependency dependency : dependencies) {
-            talendLibCoordinateMap.put(getCoordinate(dependency), dependency);
-        }
-
-        // libraries
-        dependencies.clear();
-        Set<ModuleNeeded> modules = getJobProcessor()
-                .getNeededModules(TalendProcessOptionConstants.MODULES_WITH_JOBLET
-                        | TalendProcessOptionConstants.MODULES_EXCLUDE_SHADED);
-        for (ModuleNeeded module : modules) {
-            String mavenUri = module.getMavenUri();
-            Dependency dependency = PomUtil.createModuleDependency(mavenUri);
-            dependencies.add(dependency);
-        }
-        for (Dependency dependency : dependencies) {
-            if (MavenConstants.PACKAGING_POM.equals(dependency.getType())) {
-                continue;
-            }
-            String dependencyGroupId = dependency.getGroupId();
-            String coordinate = getCoordinate(dependency);
-            if (!jobCoordinateMap.containsKey(coordinate)) {
-                if (MavenConstants.DEFAULT_LIB_GROUP_ID.equals(dependencyGroupId)) {
-                    talendLibCoordinateMap.put(coordinate, dependency);
-                }
-            }
-        }
-
-        // add 3rd party libraries
-        Map<String, Dependency> _3rdDepLibMap = new HashMap<String, Dependency>();
+        Set<String> jobCoordinate = new HashSet<>();
+        Set<String> talendLibCoordinate = new HashSet<>();
+        Set<String> _3rdLibCoordinate = new HashSet<>();
         Map<String, Set<Dependency>> duplicateLibs = new HashMap<>();
-        for (Dependency dependency : dependencies) {
-            if (MavenConstants.PACKAGING_POM.equals(dependency.getType())) {
-                continue;
-            }
-            String coordinate = getCoordinate(dependency);
-            if (!jobCoordinateMap.containsKey(coordinate) && !talendLibCoordinateMap.containsKey(coordinate)) {
-                _3rdDepLibMap.put(coordinate, dependency);
-                addToDuplicateLibs(duplicateLibs, dependency);
-            }
-        }
+        IProcessor processor = getJobProcessor();
 
-        // add missing modules from the job generation of children
-        Set<ModuleNeeded> fullModulesList = new HashSet<>();
-        for (JobInfo jobInfo : childrenJobInfo) {
-            fullModulesList
-                    .addAll(LastGenerationInfo
-                            .getInstance()
-                            .getModulesNeededWithSubjobPerJob(jobInfo.getJobId(), jobInfo.getJobVersion()));
-        }
-        for (ModuleNeeded moduleNeeded : fullModulesList) {
-            if (moduleNeeded.isExcluded()) {
-                continue;
+        // current job
+        Property currentJobProperty = processor.getProperty();
+        jobCoordinate.add(getJobCoordinate(currentJobProperty));
+        
+        // children jobs without test cases
+        Set<JobInfo> childrenJobInfo = !hasLoopDependency() ? processor.getBuildChildrenJobs().stream().filter(j -> !j.isTestContainer()).collect(Collectors.toSet()) : Collections.emptySet();
+        childrenJobInfo.forEach(j -> jobCoordinate.add(getJobCoordinate(j.getProcessItem().getProperty())));
+
+        // talend libraries and codes
+        String projectGroupId = PomIdsHelper.getProjectGroupId(ProjectManager.getInstance().getProject(currentJobProperty).getTechnicalLabel());
+
+        List<Dependency> dependencies = new ArrayList<>();
+        // codes
+        addCodesDependencies(dependencies);
+
+        // codes dependencies (optional)
+        ERepositoryObjectType.getAllTypesOfCodes().forEach(t -> dependencies.addAll(PomUtil.getCodesDependencies(t)));
+
+        // libraries of talend/3rd party
+        dependencies.addAll(convertToDistinctedJobDependencies(currentJobProperty.getId(), currentJobProperty.getVersion(),
+                processor.getNeededModules(TalendProcessOptionConstants.MODULES_EXCLUDE_SHADED)));
+
+        // missing modules from the job generation of children
+        childrenJobInfo.forEach(j -> dependencies.addAll(convertToDistinctedJobDependencies(j.getJobId(), j.getJobVersion(),
+                LastGenerationInfo.getInstance().getModulesNeededPerJob(j.getJobId(), j.getJobVersion()))));
+
+        Set<ModuleNeeded> modules = new HashSet<>();
+        // testcase modules from current job (optional)
+        modules.addAll(ProcessorDependenciesManager.getTestcaseNeededModules(currentJobProperty));
+
+        // testcase modules from children job (optional)
+        childrenJobInfo.forEach(
+                j -> modules.addAll(ProcessorDependenciesManager.getTestcaseNeededModules(j.getProcessItem().getProperty())));
+
+        dependencies.addAll(
+                modules.stream().filter(m -> !m.isExcluded()).map(m -> createDenpendency(m, true)).collect(Collectors.toSet()));
+
+        dependencies.stream().filter(d -> !MavenConstants.PACKAGING_POM.equals(d.getType())).forEach(d -> {
+            String coordinate = getCoordinate(d);
+            String groupId = d.getGroupId();
+            boolean optional = ((SortableDependency) d).isAssemblyOptional();
+            if (jobCoordinate.contains(coordinate) || talendLibCoordinate.contains(coordinate)
+                    || _3rdLibCoordinate.contains(coordinate)) {
+                return;
             }
-            MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(moduleNeeded.getMavenUri());
-            String coordinate = getCoordinate(artifact.getGroupId(), artifact.getArtifactId(), artifact.getType(),
-                    artifact.getVersion());
-            if (!jobCoordinateMap.containsKey(coordinate) && !talendLibCoordinateMap.containsKey(coordinate)
-                    && !_3rdDepLibMap.containsKey(coordinate)) {
-                Dependency dependencyObject = getDependencyObject(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion(), artifact.getType(), artifact.getClassifier());
-                if (MavenConstants.DEFAULT_LIB_GROUP_ID.equals(artifact.getGroupId())
-                        || artifact.getGroupId().startsWith(projectGroupId)) {
-                    talendLibCoordinateMap.put(coordinate, dependencyObject);
-                } else {
-                    _3rdDepLibMap.put(coordinate, dependencyObject);
-                    Dependency dependency = PomUtil
-                            .createDependency(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion(),
-                                    artifact.getType(), artifact.getClassifier());
-                    addToDuplicateLibs(duplicateLibs, dependency);
+            if (MavenConstants.DEFAULT_LIB_GROUP_ID.equals(groupId) || groupId.startsWith(projectGroupId)) {
+                if (!optional) {
+                    talendLibCoordinate.add(coordinate);
                 }
+            } else {
+                if (!optional) {
+                    _3rdLibCoordinate.add(coordinate);
+                }
+                addToDuplicateLibs(duplicateLibs, d);
             }
-        }
+        });
 
         Iterator<String> iterator = duplicateLibs.keySet().iterator();
         while (iterator.hasNext()) {
-            String key = iterator.next();
-            Set<Dependency> dupDependencies = duplicateLibs.get(key);
-            if (dupDependencies.size() < 2) {
-                // remove non-duplicated dependencies
+            Set<Dependency> dupDependencies = duplicateLibs.get(iterator.next());
+            if (dupDependencies.size() < 2 // remove unique dependency
+            /* || dupDependencies.stream().filter(d -> !((SortableDependency) d).isAssemblyOptional()).count() == 1 */) {
+                // remove when only one required dependencies, means others are from codes/testcase
+                // don't do this now at least it won't have problem in studio
+                // in some case, the needed jar is not in main job pom, maven will get the nearest one which could be
+                // wrong
                 iterator.remove();
             } else {
-                // remove duplicated dependencies from 3rd lib list
-                for (Dependency dependency : dupDependencies) {
-                    _3rdDepLibMap.remove(getCoordinate(dependency));
-                }
+                // remove duplicate dependencies from 3rd party libs
+                dupDependencies.stream().map(d -> getCoordinate(d)).forEach(c -> _3rdLibCoordinate.remove(c));
             }
         }
 
         try {
             Document document = PomUtil.loadAssemblyFile(null, assemblyFile);
             // add talend libs & codes
-            setupDependencySetNode(document, talendLibCoordinateMap, "lib", "${artifact.artifactId}.${artifact.extension}",
+            setupDependencySetNode(document, talendLibCoordinate, "lib", "${artifact.artifactId}.${artifact.extension}", false,
+                    false);
+            // add 3rd party libs: groupId:artifactId:type:version
+            setupDependencySetNode(document,
+                    _3rdLibCoordinate.stream().filter(s -> s.split(":").length == 4).collect(Collectors.toSet()), "lib", null,
                     false, false);
-            // add 3rd party libs <dependencySet>
-            setupDependencySetNode(document, _3rdDepLibMap, "lib", null, false, false);
+            // add 3rd party libs with classifier: groupId:artifactId:type:classifier:version
+            setupDependencySetNode(document,
+                    _3rdLibCoordinate.stream().filter(s -> s.split(":").length == 5).collect(Collectors.toSet()), "lib", null,
+                    false, false);
+            // FIXME if later add classifier for org.talend.libraries libs, code and job artifact, need to handle it
+            // like 3rd libs as well
+
             // add jobs
-            setupDependencySetNode(document, jobCoordinateMap, "${talend.job.name}",
+            setupDependencySetNode(document, jobCoordinate, "${talend.job.name}",
                     "${artifact.build.finalName}.${artifact.extension}", true, false);
             // add duplicate dependencies if exists
-            setupFileNode(document, duplicateLibs);
+            setupFileNode(document, duplicateLibs.values().stream().flatMap(s -> s.stream()).collect(Collectors.toSet()));
 
             PomUtil.saveAssemblyFile(assemblyFile, document);
         } catch (Exception e) {
@@ -760,66 +731,90 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         }
     }
 
+    // remove duplicate job dependencies and only keep the latest one
+    // keep high priority dependencies if set by tLibraryLoad
+    private Set<Dependency> convertToDistinctedJobDependencies(String jobId, String jobVersion, Set<ModuleNeeded> neededModules) {
+        Set<Dependency> highPriorityDependencies = LastGenerationInfo.getInstance()
+                .getHighPriorityModuleNeededPerJob(jobId, jobVersion).stream().map(m -> createDenpendency(m, false))
+                .collect(Collectors.toSet());
+        Map<String, Dependency> highPriorityDependenciesMap = new HashMap<>();
+        highPriorityDependencies.forEach(d -> highPriorityDependenciesMap.putIfAbsent(getCheckDupCoordinate(d), d));
+
+        Set<Dependency> jobDependencies = neededModules.stream().filter(m -> !m.isExcluded()).map(m -> createDenpendency(m, false)).collect(Collectors.toSet());
+        Map<String, Set<Dependency>> jobDependenciesMap = new HashMap<>();
+        jobDependencies.forEach(d -> {
+            String coordinate = getCheckDupCoordinate(d);
+            if (jobDependenciesMap.get(coordinate) == null) {
+                jobDependenciesMap.put(coordinate, new LinkedHashSet<>());
+            }
+            jobDependenciesMap.get(coordinate).add(d);
+        });
+
+        Set<Dependency> filteredDependencies = new HashSet<>();
+        jobDependenciesMap.forEach((key, value) -> {
+            Optional<Dependency> target = null;
+            if (highPriorityDependenciesMap.containsKey(key)) {
+                Dependency highPriorityDependency = highPriorityDependenciesMap.get(key);
+                target = value.stream().filter(d -> getCoordinate(highPriorityDependency).equals(getCoordinate(d))).findFirst();
+            } else {
+                target = value.stream().sorted(
+                        (d1, d2) -> new ComparableVersion(d2.getVersion()).compareTo(new ComparableVersion(d1.getVersion())))
+                        .findFirst();
+            }
+            if (target.isPresent()) {
+                filteredDependencies.add(target.get());
+            }
+        });
+
+        return filteredDependencies;
+    }
+
+    private Dependency createDenpendency(ModuleNeeded moduleNeeded, boolean optional) {
+        SortableDependency dependency = (SortableDependency) PomUtil.createModuleDependency(moduleNeeded.getMavenUri());
+        dependency.setAssemblyOptional(optional);
+        return dependency;
+    }
+
     private String getCoordinate(Dependency dependency) {
         return getCoordinate(dependency.getGroupId(), dependency.getArtifactId(), dependency.getType(),
-                dependency.getVersion());
+                dependency.getVersion(), dependency.getClassifier());
     }
 
-    protected String getCoordinate(String groupId, String artifactId, String type, String version) {
-        String separator = ":"; //$NON-NLS-1$
-        String coordinate = groupId + separator;
-        coordinate += artifactId + separator;
-        if (type != null) {
-            coordinate += type;
-        }
+    private String getCheckDupCoordinate(Dependency dependency) {
+        return getCoordinate(dependency.getGroupId(), dependency.getArtifactId(), dependency.getType(), null,
+                dependency.getClassifier());
+    }
 
-        if (version != null) {
+    protected String getJobCoordinate(Property property) {
+        return getCoordinate(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(property),
+                MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(property), null);
+    }
+
+    protected String getCoordinate(String groupId, String artifactId, String type, String version, String classifier) {
+        String separator = ":"; //$NON-NLS-1$
+        String coordinate = groupId + separator + artifactId;
+        if (StringUtils.isNotBlank(type)) {
+            coordinate += separator + type;
+        }
+        if (StringUtils.isNotBlank(classifier)) {
+            coordinate += separator + classifier;
+        }
+        if (StringUtils.isNotBlank(version)) {
             coordinate += separator + version;
         }
-
         return coordinate;
     }
 
-    protected String getAssemblyCoordinate(Dependency dependency) {
-        String separator = ":"; //$NON-NLS-1$
-        String coordinate = dependency.getGroupId() + separator;
-        coordinate += dependency.getArtifactId() + separator;
-        if (dependency.getType() != null) {
-            coordinate += dependency.getType();
-        }
-        if (dependency.getClassifier() != null) {
-            coordinate += separator + "*";
-        }
-        if (dependency.getVersion() != null) {
-            coordinate += separator + dependency.getVersion();
-        }
-
-        return coordinate;
-    }
-
-    protected Dependency getDependencyObject(String groupId, String artifactId, String version, String type, String classifier) {
-        Dependency object = new SortableDependency();
-        object.setGroupId(groupId);
-        object.setArtifactId(artifactId);
-        object.setVersion(version);
-        object.setType(type);
-        object.setClassifier(classifier);
-
-        return object;
-    }
-
     private void addToDuplicateLibs(Map<String, Set<Dependency>> map, Dependency dependency) {
-        String coordinate =
-                getCoordinate(dependency.getGroupId(), dependency.getArtifactId(), dependency.getType(), null);
+        String coordinate = getCheckDupCoordinate(dependency);
         if (!map.containsKey(coordinate)) {
-            Set<Dependency> set = new HashSet<>();
-            map.put(coordinate, set);
+            map.put(coordinate, new HashSet<>());
         }
         map.get(coordinate).add(dependency);
     }
 
-    protected void setupDependencySetNode(Document document, Map<String, Dependency> libIncludes, String outputDir,
-            String fileNameMapping, boolean useProjectArtifact, boolean unpack) {
+    protected void setupDependencySetNode(Document document, Set<String> libIncludes, String outputDir, String fileNameMapping,
+            boolean useProjectArtifact, boolean unpack) {
         if (libIncludes.isEmpty()) {
             return;
         }
@@ -839,9 +834,9 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         Node includesNode = document.createElement("includes");
         dependencySetNode.appendChild(includesNode);
 
-        for (Dependency dependency : libIncludes.values()) {
+        for (String coodinate : libIncludes) {
             Node includeNode = document.createElement("include");
-            includeNode.setTextContent(getAssemblyCoordinate(dependency));
+            includeNode.setTextContent(coodinate);
             includesNode.appendChild(includeNode);
         }
 
@@ -863,44 +858,40 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
 
     }
 
-    private void setupFileNode(Document document, Map<String, Set<Dependency>> duplicateDependencies) {
+    private void setupFileNode(Document document, Set<Dependency> duplicateDependencies) throws CoreException {
+        Node filesNode = document.getElementsByTagName("files").item(0);
+        // TESB-27614:NPE while building a route
+        if (filesNode == null) {
+            return;
+        }
         if (duplicateDependencies.isEmpty()) {
             return;
         }
-        try {
-            IMaven maven = MavenPlugin.getMaven();
-            ArtifactRepository repository = maven.getLocalRepository();
-            Node filesNode = document.getElementsByTagName("files").item(0);
-            // TESB-27614:NPE while building a route
-            if (filesNode != null) {
-                for (Entry<String, Set<Dependency>> entry : duplicateDependencies.entrySet()) {
-                    Set<Dependency> dependencies = entry.getValue();
-                    for (Dependency dependency : dependencies) {
-                        String sourceLocation = maven
-                                .getArtifactPath(repository, dependency.getGroupId(), dependency.getArtifactId(),
-                                        dependency.getVersion(), dependency.getType(), dependency.getClassifier());
-                        Path path = new File(repository.getBasedir()).toPath().resolve(sourceLocation);
-                        sourceLocation = path.toString();
-                        String destName = path.getFileName().toString();
-                        Node fileNode = document.createElement("file");
-                        filesNode.appendChild(fileNode);
-    
-                        Node sourcesNode = document.createElement("source");
-                        sourcesNode.setTextContent(sourceLocation);
-                        fileNode.appendChild(sourcesNode);
-    
-                        Node outputDirNode = document.createElement("outputDirectory");
-                        outputDirNode.setTextContent("lib");
-                        fileNode.appendChild(outputDirNode);
-    
-                        Node destNameNode = document.createElement("destName");
-                        destNameNode.setTextContent(destName);
-                        fileNode.appendChild(destNameNode);
-                    }
-                }
+        IMaven maven = MavenPlugin.getMaven();
+        ArtifactRepository repository = maven.getLocalRepository();
+        for (Dependency dependency : duplicateDependencies) {
+            if (((SortableDependency) dependency).isAssemblyOptional()) {
+                continue;
             }
-        } catch (CoreException e) {
-            ExceptionHandler.process(e);
+            String sourceLocation = maven.getArtifactPath(repository, dependency.getGroupId(), dependency.getArtifactId(),
+                    dependency.getVersion(), dependency.getType(), dependency.getClassifier());
+            Path path = new File(repository.getBasedir()).toPath().resolve(sourceLocation);
+            sourceLocation = path.toString();
+            String destName = path.getFileName().toString();
+            Node fileNode = document.createElement("file");
+            filesNode.appendChild(fileNode);
+
+            Node sourcesNode = document.createElement("source");
+            sourcesNode.setTextContent(sourceLocation);
+            fileNode.appendChild(sourcesNode);
+
+            Node outputDirNode = document.createElement("outputDirectory");
+            outputDirNode.setTextContent("lib");
+            fileNode.appendChild(outputDirNode);
+
+            Node destNameNode = document.createElement("destName");
+            destNameNode.setTextContent(destName);
+            fileNode.appendChild(destNameNode);
         }
     }
 

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenStandardJobOSGiPom.java

@@ -17,16 +17,14 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.maven.model.Dependency;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.Plugin;
-import org.apache.maven.model.PluginExecution;
 import org.apache.maven.model.Profile;
-import org.codehaus.plexus.util.xml.Xpp3Dom;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.runtime.IProgressMonitor;
@@ -45,7 +43,6 @@ import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.repository.model.ProxyRepositoryFactory;
-import org.talend.core.runtime.maven.MavenConstants;
 import org.talend.core.runtime.process.ITalendProcessJavaProject;
 import org.talend.core.runtime.process.TalendProcessArgumentConstant;
 import org.talend.core.runtime.process.TalendProcessOptionConstants;
@@ -54,7 +51,6 @@ import org.talend.core.runtime.projectsetting.IProjectSettingTemplateConstants;
 import org.talend.designer.maven.model.TalendMavenConstants;
 import org.talend.designer.maven.template.MavenTemplateManager;
 import org.talend.designer.maven.tools.AggregatorPomsHelper;
-import org.talend.designer.maven.utils.PomIdsHelper;
 import org.talend.designer.maven.utils.PomUtil;
 import org.talend.designer.runprocess.IProcessor;
 import org.talend.repository.model.IProxyRepositoryFactory;
@@ -190,31 +186,22 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
 
     @Override
     protected void updateDependencySet(IFile assemblyFile) {
-        Map<String, Dependency> jobCoordinateMap = new HashMap<String, Dependency>();
+        Set<String> jobCoordinate = new HashSet<>();
         if (!hasLoopDependency()) {
             // add children jobs
             Set<JobInfo> childrenJobInfo = getJobProcessor().getBuildChildrenJobs();
             for (JobInfo jobInfo : childrenJobInfo) {
-                Property property = jobInfo.getProcessItem().getProperty();
-                String coordinate = getCoordinate(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo),
-                        MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(property));
-                Dependency dependency = getDependencyObject(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo), PomIdsHelper.getJobVersion(property),
-                                MavenConstants.PACKAGING_JAR, null);
-                jobCoordinateMap.put(coordinate, dependency);
+                jobCoordinate.add(getJobCoordinate(jobInfo.getProcessItem().getProperty()));
             }
         }
-        // add parent job
-        Property parentProperty = this.getJobProcessor().getProperty();
-        String parentCoordinate = getCoordinate(PomIdsHelper.getJobGroupId(parentProperty),
-                PomIdsHelper.getJobArtifactId(parentProperty), MavenConstants.PACKAGING_JAR,
-                PomIdsHelper.getJobVersion(parentProperty));
-        Dependency parentDependency = getDependencyObject(PomIdsHelper.getJobGroupId(parentProperty), PomIdsHelper.getJobArtifactId(parentProperty), PomIdsHelper.getJobVersion(parentProperty),
-                        MavenConstants.PACKAGING_JAR, null);
-        jobCoordinateMap.put(parentCoordinate, parentDependency);
+        // add current job
+        Property currentJobProperty = getJobProcessor().getProperty();
+        String parentCoordinate = getJobCoordinate(currentJobProperty);
+        jobCoordinate.add(parentCoordinate);
         try {
             Document document = PomUtil.loadAssemblyFile(null, assemblyFile);
             // add jobs
-            setupDependencySetNode(document, jobCoordinateMap, null, "${artifact.build.finalName}.${artifact.extension}", true,
+            setupDependencySetNode(document, jobCoordinate, null, "${artifact.build.finalName}.${artifact.extension}", true,
                     true);
             PomUtil.saveAssemblyFile(assemblyFile, document);
         } catch (Exception e) {

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/PomUtil.java

@@ -34,6 +34,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -81,6 +82,7 @@ import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.relationship.Relation;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
+import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.nexus.TalendMavenResolver;
@@ -96,6 +98,7 @@ import org.talend.designer.maven.template.MavenTemplateManager;
 import org.talend.designer.maven.tools.AggregatorPomsHelper;
 import org.talend.designer.maven.tools.ProcessorDependenciesManager;
 import org.talend.designer.runprocess.IProcessor;
+import org.talend.designer.runprocess.IRunProcessService;
 import org.talend.repository.ProjectManager;
 import org.talend.utils.xml.XmlUtils;
 import org.w3c.dom.Attr;
@@ -1133,4 +1136,20 @@ public class PomUtil {
         }
         return found;
     }
+
+    public static Set<Dependency> getCodesDependencies(ERepositoryObjectType codeType) {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
+            IRunProcessService runProcessService = GlobalServiceRegister.getDefault().getService(IRunProcessService.class);
+            try {
+                Model model = MODEL_MANAGER.readMavenModel(runProcessService.getTalendCodeJavaProject(codeType).getProjectPom());
+                return model.getDependencies().stream().map(
+                        d -> createDependency(d.getGroupId(), d.getArtifactId(), d.getVersion(), d.getType(), d.getClassifier()))
+                        .peek(d -> ((SortableDependency) d).setAssemblyOptional(true)).collect(Collectors.toSet());
+            } catch (CoreException e) {
+                ExceptionHandler.process(e);
+            }
+        }
+        return Collections.emptySet();
+    }
+
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/SortableDependency.java

@@ -1,13 +1,80 @@
 package org.talend.designer.maven.utils;
 
+import org.apache.maven.artifact.versioning.ComparableVersion;
 import org.apache.maven.model.Dependency;
 
 public class SortableDependency extends Dependency implements Comparable<SortableDependency> {
 
+    private static final long serialVersionUID = -6295226523517981508L;
+
+    /**
+     * dependencies in testcase/routines/beans no need to be assembled in final zip.
+     */
+    private boolean isAssemblyOptional;
+
     @Override
     public int compareTo(SortableDependency o) {
+        int compare = getArtifactId().compareTo(o.getArtifactId());
+        if (compare == 0) {
+            // FIXME according to Maven official Doc for dependencies:
+            // https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html
+            // when pom has duplicate version of dependencies, Maven should take the first one
+            // but in practice(maven 3.5.3 embedded), it always take the last one
+            // we only need job dependencies in final job zip rather than testcase's
+            // so we always put the latest job dependency at the bottom
+            // if maven fix it in future, we need to reverse the order as well.
+            if (isAssemblyOptional && !o.isAssemblyOptional) {
+                return -1;
+            }
+            if (!isAssemblyOptional && o.isAssemblyOptional) {
+                return 1;
+            }
+            return new ComparableVersion(getVersion()).compareTo(new ComparableVersion(o.getVersion()));
+        }
+        return compare;
+    }
 
-        return getArtifactId().compareTo(o.getArtifactId());
+    public boolean isAssemblyOptional() {
+        return isAssemblyOptional;
+    }
+
+    public void setAssemblyOptional(boolean isAssemblyOptional) {
+        this.isAssemblyOptional = isAssemblyOptional;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 17;
+        result = 31 * result + (getGroupId() == null ? 0 : getGroupId().hashCode());
+        result = 31 * result + (getArtifactId() == null ? 0 : getArtifactId().hashCode());
+        result = 31 * result + (getVersion() == null ? 0 : getVersion().hashCode());
+        result = 31 * result + (getType() == null ? 0 : getType().hashCode());
+        result = 31 * result + (getClassifier() == null ? 0 : getClassifier().hashCode());
+        result = 31 * result + (isAssemblyOptional() ? 1231 : 1237);
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (!(obj instanceof SortableDependency)) {
+            return false;
+        }
+        SortableDependency caseobj = (SortableDependency) obj;
+
+        return (getGroupId() == caseobj.getGroupId() || (getGroupId() != null && getGroupId().equals(caseobj.getGroupId())))
+                && (getArtifactId() == caseobj.getArtifactId()
+                        || (getArtifactId() != null && getArtifactId().equals(caseobj.getArtifactId())))
+                && (getVersion() == caseobj.getVersion() || (getVersion() != null && getVersion().equals(caseobj.getVersion())))
+                && (getType() == caseobj.getType() || (getType() != null && getType().equals(caseobj.getType())))
+                && (getClassifier() == caseobj.getClassifier()
+                        || (getClassifier() != null && getClassifier().equals(caseobj.getClassifier())))
+                && (isAssemblyOptional() == caseobj.isAssemblyOptional());
     }
 
 }

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/ui/dialogs/ExternalModulesInstallDialog.java

@@ -85,6 +85,7 @@ import org.talend.core.model.general.ILibrariesService;
 import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
 import org.talend.core.model.general.ModuleStatusProvider;
 import org.talend.core.model.general.ModuleToInstall;
+import org.talend.core.nexus.TalendLibsServerManager;
 import org.talend.core.runtime.maven.MavenConstants;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.librariesmanager.ui.LibManagerUiPlugin;
@@ -147,7 +148,8 @@ public class ExternalModulesInstallDialog extends TitleAreaDialog implements IMo
     }
 
     private void checkNetworkStatus() {
-        boolean networkValid = NetworkUtil.isNetworkValid(System.getProperty("nexus.proxy.url"));
+        boolean networkValid = NetworkUtil
+                .isNetworkValid(TalendLibsServerManager.getInstance().getTalentArtifactServer().getServer());
         if (!networkValid) {
             setErrorMessage(Messages.getString("ExternalModulesInstallDialog.networkUnavailable.msg")); //$NON-NLS-1$
         }

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/utils/RemoteModulesHelper.java

@@ -148,7 +148,8 @@ public class RemoteModulesHelper {
 
             Set<String> unavailableModules = new HashSet<String>();
             // if the network is not valid, all jars are not available.
-            boolean networkValid = NetworkUtil.isNetworkValid(System.getProperty("nexus.proxy.url"));
+            boolean networkValid = NetworkUtil
+                    .isNetworkValid(TalendLibsServerManager.getInstance().getTalentArtifactServer().getServer());
             if (!networkValid) {
                 if (!alreadyWarnedAboutConnectionIssue) {
                     log.warn("failed to connect to internet");

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/wizard/metadata/ContextSetsSelectionDialog.java

@@ -67,13 +67,25 @@ public class ContextSetsSelectionDialog extends SelectionDialog {
         initSets();
     }
 
-    /**
-     * DOC xqliu Comment method "initDialog".
-     *
-     * @param object
-     * @param cancelFlag
-     * @param message
-     */
+    public ContextSetsSelectionDialog(ContextItem contextItem) {
+        this(calcShell(), contextItem, false);
+    }
+
+    public ContextSetsSelectionDialog(List<ContextType> contexts, String defaultContextName, boolean canCancel) {
+        super(calcShell());
+        initDialog(contexts, canCancel, Messages.getString("ContextSetsSelectionDialog.ReportMessages")); //$NON-NLS-1$
+        this.defalutContext = defaultContextName;
+        initSets();
+    }
+
+    public ContextSetsSelectionDialog(Shell parentShell, Object source, boolean canCancel, String defaultContextName,
+            String message) {
+        super(parentShell == null ? calcShell() : parentShell);
+        initDialog(source, canCancel, message); // $NON-NLS-1$
+        this.defalutContext = defaultContextName;
+        initSets();
+    }
+
     private void initDialog(Object object, boolean cancelFlag, String message) {
         this.source = object;
         this.canCancel = cancelFlag;
@@ -89,17 +101,6 @@ public class ContextSetsSelectionDialog extends SelectionDialog {
         newShell.setImage(ImageProvider.getImage(ECoreImage.CONTEXT_ICON));
     }
 
-    public ContextSetsSelectionDialog(ContextItem contextItem) {
-        this(calcShell(), contextItem, false);
-    }
-
-    public ContextSetsSelectionDialog(List<ContextType> contexts, String defaultContextName, boolean canCancel) {
-        super(calcShell());
-        initDialog(contexts, canCancel, Messages.getString("ContextSetsSelectionDialog.ReportMessages")); //$NON-NLS-1$
-        this.defalutContext = defaultContextName;
-        initSets();
-    }
-
     /**
      * DOC zli Comment method "calcShell".
      *

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/ExtractMetaDataUtils.java

@@ -1039,7 +1039,7 @@ public class ExtractMetaDataUtils {
                     if (driverJarPathArg.contains(";")) {
                         String jars[] = driverJarPathArg.split(";");
                         for (String jar : jars) {
-                            String jarName = librairesManagerService.getJarNameFromMavenuri(driverJarPathArg);
+                            String jarName = librairesManagerService.getJarNameFromMavenuri(jar);
                             // TDQ-16842 msjian:sometimes for the import jdbc connection, the jarName is null
                             if (jarName == null) {
                                 jarName = jar.split("/")[1] + ".jar";

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/metadata/managment/model/DBConnectionFillerImpl.java

@@ -445,7 +445,8 @@ public class DBConnectionFillerImpl extends MetadataFillerImpl<DatabaseConnectio
             ResultSet catalogNames = null;
             if (dbJDBCMetadata instanceof SybaseDatabaseMetaData) {
                 // Whether in context mode or not, metaConnection can get the correct username always
-                catalogNames = ((SybaseDatabaseMetaData) dbJDBCMetadata).getCatalogs(metaConnection.getUsername());
+                String username = metaConnection == null ? dbConn.getUsername() : metaConnection.getUsername();
+                catalogNames = ((SybaseDatabaseMetaData) dbJDBCMetadata).getCatalogs(username);
             } else {
                 catalogNames = dbJDBCMetadata.getCatalogs();
             }

main/plugins/org.talend.repository.items.importexport/src/main/java/org/talend/repository/items/importexport/handlers/exports/SyncChildrenSourceCodeProvider.java

@@ -31,6 +31,13 @@ public class SyncChildrenSourceCodeProvider implements IBuildResourcesProvider {
 
     @Override
     public void prepare(IProgressMonitor monitor, Map<String, Object> parameters) throws Exception {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
+            IRunProcessService service = GlobalServiceRegister.getDefault().getService(IRunProcessService.class);
+            if (service.isCIMode()) {
+                // don't sync any children source code to parent job in CI.
+                return;
+            }
+        }
         if (parameters == null) {
             return;
         }

main/plugins/org.talend.repository.view/src/org/talend/repository/navigator/RepoViewCommonViewer.java

@@ -444,4 +444,21 @@ public class RepoViewCommonViewer extends CommonViewer implements INavigatorCont
         super.dispose();
     }
 
+    @Override
+    public void associate(Object element, Item item) {
+        Object data = item.getData();
+        if (data != element) {
+            if (data != null) {
+                disassociate(item);
+            }
+            item.setData(element);
+            mapElement(element, item);
+        } else {
+            // Always map the element, even if data == element,
+            // since unmapAllElements() can leave the map inconsistent
+            // See bug 2741 for details.
+            mapElement(element, item);
+        }
+    }
+
 }

main/plugins/org.talend.updates.runtime/src/main/java/org/talend/updates/runtime/engine/factory/PluginOptionalMissingJarsExtraUpdatesFactory.java

@@ -24,6 +24,7 @@ import org.eclipse.jface.operation.IRunnableWithProgress;
 import org.talend.commons.utils.network.NetworkUtil;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.ModuleToInstall;
+import org.talend.core.nexus.TalendLibsServerManager;
 import org.talend.librariesmanager.model.ModulesNeededProvider;
 import org.talend.librariesmanager.utils.RemoteModulesHelper;
 import org.talend.updates.runtime.i18n.Messages;
@@ -38,6 +39,7 @@ public class PluginOptionalMissingJarsExtraUpdatesFactory extends AbstractExtraU
 
     private static Logger log = Logger.getLogger(PluginOptionalMissingJarsExtraUpdatesFactory.class);
 
+
     /*
      * (non-Javadoc)
      *
@@ -48,7 +50,7 @@ public class PluginOptionalMissingJarsExtraUpdatesFactory extends AbstractExtraU
     @Override
     public void retrieveUninstalledExtraFeatures(IProgressMonitor monitor, Set<ExtraFeature> uninstalledExtraFeatures)
             throws Exception {
-        if (!NetworkUtil.isNetworkValid(System.getProperty("nexus.proxy.url"))) {
+        if (!NetworkUtil.isNetworkValid(TalendLibsServerManager.getInstance().getTalentArtifactServer().getServer())) {
     		return;
     	}
     	
@@ -94,7 +96,5 @@ public class PluginOptionalMissingJarsExtraUpdatesFactory extends AbstractExtraU
                                 .getString("missing.optional.third.parties.libs.feature.description"), false/* mustInstall */)); //$NON-NLS-1$
             }
         }// else nothing to install so nothing to install ;)
-
     }
-
-}
+}

test/plugins/org.talend.core.runtime.test/META-INF/MANIFEST.MF

@@ -9,3 +9,4 @@ Require-Bundle: org.junit;bundle-version="4.11.0",
  org.talend.testutils,
  org.talend.components.common,
  javax.xml.soap
+Import-Package: org.talend.core.repository.model

test/plugins/org.talend.core.runtime.test/src/org/talend/core/model/utils/RepositoryNodeTest.java

@@ -0,0 +1,56 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.utils;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+
+import org.junit.Test;
+import org.talend.core.model.repository.ERepositoryObjectType;
+import org.talend.repository.model.IRepositoryNode.ENodeType;
+import org.talend.repository.model.IRepositoryNode.EProperties;
+import org.talend.repository.model.RepositoryNode;
+
+/*
+ * Created by bhe on Mar 30, 2020
+ */
+public class RepositoryNodeTest {
+
+    @Test
+    public void testEquals() throws Exception {
+        RepositoryNode parent = new RepositoryNode(null, null, ENodeType.STABLE_SYSTEM_FOLDER);
+        parent.setProperties(EProperties.LABEL, "system");
+        parent.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.PROCESS);
+        parent.setType(ENodeType.STABLE_SYSTEM_FOLDER);
+
+        assertNotNull(parent);
+
+        RepositoryNode parentNew = new RepositoryNode(null, null, ENodeType.STABLE_SYSTEM_FOLDER);
+        parentNew.setProperties(EProperties.LABEL, "system");
+        parentNew.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.PROCESS);
+        parentNew.setType(ENodeType.STABLE_SYSTEM_FOLDER);
+
+        assertEquals(parent, parentNew);
+        assertEquals(parent.hashCode(), parentNew.hashCode());
+
+        RepositoryNode son = new RepositoryNode(null, null, ENodeType.STABLE_SYSTEM_FOLDER);
+        son.setProperties(EProperties.LABEL, ERepositoryObjectType.PROCESS);
+        son.setProperties(EProperties.CONTENT_TYPE, ERepositoryObjectType.PROCESS);
+
+        assertNotNull(son);
+
+        assertFalse(parent.equals(son));
+
+    }
+}

test/plugins/org.talend.designer.maven.test/src/main/java/org/talend/designer/maven/utils/SortableDependencyTest.java

@@ -0,0 +1,49 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.designer.maven.utils;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * DOC jding class global comment. Detailled comment
+ */
+public class SortableDependencyTest {
+
+    @Test
+    public void testEqualsHashcode() {
+        SortableDependency dependency = new SortableDependency();
+        dependency.setGroupId("org.example.test");
+        dependency.setArtifactId("test");
+        SortableDependency dependency1 = new SortableDependency();
+        dependency1.setGroupId("org.example.test");
+        dependency1.setArtifactId("test");
+
+        Set<SortableDependency> dependencySet = new HashSet<SortableDependency>();
+        dependencySet.add(dependency);
+        dependencySet.add(dependency1);
+        Assert.assertEquals(1, dependencySet.size());
+        Assert.assertTrue(dependency.equals(dependency1));
+
+        dependency.setVersion("0.1");
+        dependency1.setVersion("0.2");
+        dependencySet.add(dependency1);
+        Assert.assertEquals(2, dependencySet.size());
+        Assert.assertFalse(dependency.equals(dependency1));
+
+    }
+
+}