fix(TESB-28815): Fix bundle name (should include -bundle suffix at the end)

* fix(TUP-26081): Share libraries pb

https://jira.talendforge.org/browse/TUP-26081

* fix(TUP-26081): Share libraries pb

https://jira.talendforge.org/browse/TUP-26081

Co-authored-by: Chao MENG <cmeng@talend.com>

main/plugins/org.talend.commons.runtime/META-INF/MANIFEST.MF

@@ -13,7 +13,8 @@ Require-Bundle: org.apache.log4j;visibility:=reexport,
  org.talend.libraries.apache,
  com.ibm.icu,
  org.apache.commons.io,
- org.talend.utils
+ org.talend.utils,
+ org.eclipse.core.net
 Export-Package: org.talend.commons,
  org.talend.commons.exception,
  org.talend.commons.i18n,

main/plugins/org.talend.commons.runtime/src/org/talend/commons/CommonsPlugin.java

@@ -17,6 +17,7 @@ import java.io.InputStream;
 import java.net.URL;
 
 import org.apache.commons.lang.ArrayUtils;
+import org.eclipse.core.net.proxy.IProxyService;
 import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.preferences.InstanceScope;
@@ -24,6 +25,7 @@ import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleActivator;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.ServiceReference;
+import org.osgi.util.tracker.ServiceTracker;
 import org.talend.commons.exception.ExceptionService;
 import org.talend.commons.runtime.debug.TalendDebugHandler;
 import org.talend.commons.utils.system.EclipseCommandLine;
@@ -55,6 +57,8 @@ public class CommonsPlugin implements BundleActivator {
     // TESB-17856: For commandline builds ESB Micorservice bundle
     private static boolean isESBMicorservice = false;
 
+    private static ServiceTracker proxyTracker;
+
     public static boolean isWorkbenchCreated() {
         return isWorkbenchCreated;
     }
@@ -176,4 +180,13 @@ public class CommonsPlugin implements BundleActivator {
         return FileLocator.toFileURL(entry);
     }
 
+    public static IProxyService getProxyService() {
+        if (proxyTracker == null) {
+            proxyTracker = new ServiceTracker(Platform.getBundle(PLUGIN_ID).getBundleContext(), IProxyService.class.getName(),
+                    null);
+            proxyTracker.open();
+        }
+        return (IProxyService) proxyTracker.getService();
+    }
+
 }

main/plugins/org.talend.commons.runtime/src/org/talend/commons/i18n/internal/messages.properties

@@ -58,3 +58,6 @@ AS400ResultSet.unknowCloumn=Invalid argument\: unknown column name
 AS400ResultSet.parameterIndex=Invalid argument\: parameter index
 AS400ResultSet.outofRange=\ is out of range.
 ITaCoKitService.exception.multipleInstance=More than one instance found: {0}
+TalendProxySelector.exception.badUriMap=Bad uri map: {0}
+TalendProxySelector.exception.proxySelectionError=Error occurs when selecting proxy for {0}
+

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/network/TalendProxySelector.java

@@ -13,37 +13,211 @@
 package org.talend.commons.utils.network;
 
 import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.net.PasswordAuthentication;
 import java.net.Proxy;
 import java.net.ProxySelector;
 import java.net.SocketAddress;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Priority;
+import org.eclipse.core.internal.net.ProxyManager;
+import org.eclipse.core.net.proxy.IProxyService;
+import org.talend.commons.CommonsPlugin;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.i18n.internal.Messages;
+import org.talend.daikon.sandbox.properties.ClassLoaderIsolatedSystemProperties;
+
+import sun.net.spi.DefaultProxySelector;
 
 /**
  * DOC cmeng  class global comment. Detailled comment
  */
 public class TalendProxySelector extends ProxySelector {
 
+    public static final String PROP_PRINT_LOGS = "talend.studio.proxy.printLogs";
+
     private static final String ECLIPSE_PROXY_SELECTOR = ".EclipseProxySelector"; //$NON-NLS-1$
 
-    private ProxySelector defaultSelector;
+    private static final String PROP_ALLOW_PROXY_REDIRECT = "talend.studio.proxy.allowProxyRedirect";
 
-    final private List<IProxySelectorProvider> selectorProviders;
+    private static final String PROP_ALLOW_PROXY_REDIRECT_EXCLUDE = "talend.studio.proxy.redirect.whiteList";
+
+    private static final String PROP_EXECUTE_CONNECTION_FAILED = "talend.studio.proxy.executeConnectionFailed";
+
+    private static final String PROP_UPDATE_SYSTEM_PROPERTIES_FOR_JRE = "talend.studio.proxy.jre.updateSystemProperties";
+
+    private static final String PROP_CHECK_PROXY = "talend.studio.proxy.checkProxy";
+
+    private static final String PROP_VALIDATE_URI = "talend.studio.proxy.validateUri";
+
+    private static final String PROP_PROXY_SELECTOR = "talend.studio.proxy.selector";
+
+    private static final String PROP_PROXY_SELECTOR_DEFAULT = "default";
+
+    private static final String PROP_PROXY_SELECTOR_JRE = "jre";
+
+    private static final String PROP_PROXY_HOST_MAP = "talend.studio.proxy.hostMap";
+
+    private static final String PROP_DISABLE_DEFAULT_SELECTOR_PROVIDER = "talend.studio.proxy.disableDefaultSelectorProvider";
+
+    /**
+     * Example: update.talend.com,socket:http,https:http;nexus.talend.com,socket,http;,socket:http
+     */
+    private static final String PROP_PROXY_MAP_HOST_DEFAULT = "";
+
+    /**
+     * Example: svn.company.com;nexus.company.com
+     */
+    private static final String PROP_ALLOW_PROXY_REDIRECT_EXCLUDE_DEFAULT = "";
+
+    private static final String KEY_DEFAULT = ":default:";
+
+    private static Field uriHostField;
+
+    private static Method proxyManagerUpdateSystemPropertiesFunc;
+
+    private static boolean checkProxy = Boolean.valueOf(System.getProperty(PROP_CHECK_PROXY, Boolean.TRUE.toString()));
+
+    /**
+     * Note: eclipse default selector may be different between TOS and TIS, TOS may use jre one, TIS may use egit one
+     */
+    private ProxySelector eclipseDefaultSelector;
+
+    private ProxySelector jreDefaultSelector;
+
+    private EProxySelector eProxySelector;
+
+    final private Map<Object, Collection<IProxySelectorProvider>> selectorProviders;
+
+    private Map<String, Map<String, String>> hostMap;
+
+    private Set<String> redirectWhiteList;
 
     private volatile static TalendProxySelector instance;
 
     private static Object instanceLock = new Object();
 
-    private TalendProxySelector(final ProxySelector defaultSelector) {
-        this.defaultSelector = defaultSelector;
-        selectorProviders = new ArrayList<>();
+    private boolean printProxyLog = false;
+
+    private boolean allowProxyRedirect = false;
+
+    private boolean disableDefaultSelectorProvider = false;
+
+    private boolean validateUri = true;
+
+    private boolean executeConnectionFailed = true;
+
+    private boolean updateSystemPropertiesForJre = true;
+
+    private TalendProxySelector(final ProxySelector eclipseDefaultSelector) {
+        this.eclipseDefaultSelector = eclipseDefaultSelector;
+        this.jreDefaultSelector = new DefaultProxySelector();
+
+        selectorProviders = Collections.synchronizedMap(new HashMap<>());
+        allowProxyRedirect = Boolean.valueOf(System.getProperty(PROP_ALLOW_PROXY_REDIRECT, Boolean.FALSE.toString()));
+        disableDefaultSelectorProvider = Boolean
+                .valueOf(System.getProperty(PROP_DISABLE_DEFAULT_SELECTOR_PROVIDER, Boolean.FALSE.toString()));
+        printProxyLog = Boolean.valueOf(System.getProperty(PROP_PRINT_LOGS, Boolean.FALSE.toString()));
+        validateUri = Boolean.valueOf(System.getProperty(PROP_VALIDATE_URI, Boolean.TRUE.toString()));
+        executeConnectionFailed = Boolean.valueOf(System.getProperty(PROP_EXECUTE_CONNECTION_FAILED, Boolean.TRUE.toString()));
+        updateSystemPropertiesForJre = Boolean
+                .valueOf(System.getProperty(PROP_UPDATE_SYSTEM_PROPERTIES_FOR_JRE, Boolean.TRUE.toString()));
+
+        switch (System.getProperty(PROP_PROXY_SELECTOR, PROP_PROXY_SELECTOR_DEFAULT).toLowerCase()) {
+        case PROP_PROXY_SELECTOR_JRE:
+            this.eProxySelector = EProxySelector.jre;
+            break;
+        default:
+            this.eProxySelector = EProxySelector.eclipse_default;
+            break;
+        }
+
+        initHostMap();
+        initRedirectList();
+    }
+
+    private void initHostMap() {
+        try {
+            hostMap = new HashMap<>();
+            String property = System.getProperty(PROP_PROXY_HOST_MAP, PROP_PROXY_MAP_HOST_DEFAULT);
+            if (StringUtils.isEmpty(property)) {
+                return;
+            }
+            String[] splits = property.split(";");
+            for (String split : splits) {
+                try {
+                    int index = split.indexOf(',');
+                    String uri = split.substring(0, index);
+                    String key = StringUtils.strip(uri);
+                    if (StringUtils.isBlank(key)) {
+                        key = KEY_DEFAULT;
+                    }
+                    key = key.toLowerCase();
+                    Map<String, String> protocolMap = hostMap.get(key);
+                    if (protocolMap == null) {
+                        protocolMap = new HashMap<>();
+                        hostMap.put(key, protocolMap);
+                    }
+                    int protocolMapIndex = index + 1;
+                    String protocolMapStr = split.substring(protocolMapIndex);
+                    String[] entry = protocolMapStr.split(",");
+                    for (String pMap : entry) {
+                        try {
+                            String[] mapEntry = pMap.split(":");
+                            if (mapEntry.length != 2) {
+                                ExceptionHandler.process(
+                                        new Exception(Messages.getString("TalendProxySelector.exception.badUriMap", pMap)));
+                                continue;
+                            }
+                            protocolMap.put(mapEntry[0].toLowerCase(), mapEntry[1].toLowerCase());
+                        } catch (Exception e) {
+                            ExceptionHandler.process(e);
+                        }
+                    }
+                } catch (Exception e) {
+                    ExceptionHandler.process(e);
+                }
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
+    private void initRedirectList() {
+        try {
+            redirectWhiteList = new HashSet<>();
+            String property = System.getProperty(PROP_ALLOW_PROXY_REDIRECT_EXCLUDE, PROP_ALLOW_PROXY_REDIRECT_EXCLUDE_DEFAULT);
+            if (StringUtils.isEmpty(property)) {
+                return;
+            }
+            String[] split = property.split(";");
+            for (String host : split) {
+                host = StringUtils.strip(host);
+                if (StringUtils.isBlank(host)) {
+                    host = KEY_DEFAULT;
+                }
+                redirectWhiteList.add(host);
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
     }
 
     public static TalendProxySelector getInstance() {
@@ -58,6 +232,18 @@ public class TalendProxySelector extends ProxySelector {
             synchronized (instanceLock) {
                 if (instance == null) {
                     instance = new TalendProxySelector(proxySelector);
+                    try {
+                        uriHostField = URI.class.getDeclaredField("host");
+                        uriHostField.setAccessible(true);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
+                    try {
+                        proxyManagerUpdateSystemPropertiesFunc = ProxyManager.class.getDeclaredMethod("updateSystemProperties");
+                        proxyManagerUpdateSystemPropertiesFunc.setAccessible(true);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
                 }
             }
         }
@@ -70,19 +256,216 @@ public class TalendProxySelector extends ProxySelector {
                     return null;
                 }
             });
-            if (instance.getDefaultProxySelector() == null
+            if (instance.getEclipseDefaultSelector() == null
                     || (proxySelector != null && proxySelector.getClass().getName().endsWith(ECLIPSE_PROXY_SELECTOR))) {
-                instance.setDefaultProxySelector(proxySelector);
+                instance.setEclipseDefaultSelector(proxySelector);
             }
         }
 
         return instance;
     }
 
+    public static void checkProxy() {
+        if (!checkProxy) {
+            return;
+        }
+        try {
+            TalendProxySelector.getInstance();
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
     @Override
     public List<Proxy> select(final URI uri) {
-        final Set<Proxy> resultFromProviders = new HashSet<>();
-        List<IProxySelectorProvider> providers = getProxySelectorProviders();
+        if (printProxyLog) {
+            ExceptionHandler.log("TalendProxySelector.select " + uri);
+        }
+        if (uri == null) {
+            return Collections.EMPTY_LIST;
+        }
+        URI validatedUri = validateUri(uri);
+        Set<Proxy> results = new LinkedHashSet<>();
+
+        try {
+            final Set<Proxy> resultFromProviders = getProxysFromProviders(validatedUri);
+            if (resultFromProviders != null && !resultFromProviders.isEmpty()) {
+                results.addAll(resultFromProviders);
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+        if (printProxyLog) {
+            ExceptionHandler.log("TalendProxySelector.resultFromProviders " + results);
+        }
+
+        ProxySelector defaultProxySelector = getDefaultProxySelector();
+        if (printProxyLog) {
+            ExceptionHandler.log("TalendProxySelector.defaultProxySelector " + defaultProxySelector);
+        }
+        if (defaultProxySelector != null) {
+            /**
+             * don't validate uri here, so that we can know whether it is an issue uri
+             */
+            URI newUri = getNewUri(validatedUri, false);
+            List<Proxy> defaultProxys = null;
+            if (validateUri && StringUtils.isBlank(newUri.getHost())) {
+                /**
+                 * If host is blank, force to use jre proxy selector to avoid the eclipse proxy selector bug
+                 */
+                defaultProxys = getJreProxySelector().select(newUri);
+            } else {
+                defaultProxys = defaultProxySelector.select(newUri);
+            }
+            if (printProxyLog) {
+                ExceptionHandler.log("TalendProxySelector.defaultProxys " + defaultProxys);
+            }
+            try {
+                results.addAll(filterProxys(validatedUri, defaultProxys));
+            } catch (Exception e) {
+                results.addAll(defaultProxys);
+                ExceptionHandler.process(e);
+            }
+        }
+        if (printProxyLog) {
+            String proxys = results.toString();
+            ExceptionHandler.log("Selected proxys for " + uri + ", " + proxys);
+            ExceptionHandler.process(new Exception("Proxy call stacks"), Priority.INFO);
+        }
+        return new LinkedList<Proxy>(results);
+    }
+
+    private URI validateUri(URI uri) {
+        if (!validateUri) {
+            return uri;
+        }
+
+        URI validatedUri = null;
+        try {
+            /**
+             * DON'T use URI.create(), MUST use the conductor which requires authority
+             */
+            validatedUri = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), uri.getQuery(), uri.getFragment());
+
+            /**
+             * Validate the host, if the host is empty, it will cause the eclipse selector to return dirrect
+             */
+            if (StringUtils.isBlank(validatedUri.getHost())) {
+                String authority = validatedUri.getAuthority();
+                if (StringUtils.isNotBlank(authority)) {
+                    // example: https://u:p@www.company.com:8081/path/a?param=b
+                    String host = null;
+                    int userInfoIndex = authority.indexOf('@');
+                    if (0 <= userInfoIndex) {
+                        authority = authority.substring(userInfoIndex + 1);
+                    }
+                    int portIndex = authority.lastIndexOf(':');
+                    if (0 <= portIndex) {
+                        host = authority.substring(0, portIndex);
+                    }
+                    try {
+                        uriHostField.set(validatedUri, host);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
+                }
+
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+            if (validatedUri == null) {
+                validatedUri = uri;
+            }
+        }
+        if (printProxyLog) {
+            ExceptionHandler.log("After validate: " + uri + " -> " + validatedUri);
+        }
+        return validatedUri;
+    }
+
+    private List<Proxy> filterProxys(final URI uri, List<Proxy> defaultProxys) {
+        List<Proxy> result = new ArrayList<>();
+        if (defaultProxys != null && !defaultProxys.isEmpty()) {
+            for (Proxy proxy : defaultProxys) {
+                SocketAddress addr = null;
+                Proxy.Type proxyType = null;
+                if (proxy != null) {
+                    proxyType = proxy.type();
+                    addr = proxy.address();
+                }
+
+                boolean redirect = true;
+                if (!allowProxyRedirect) {
+                    String host = uri.getHost();
+                    if (host == null) {
+                        host = "";
+                    }
+                    host = StringUtils.strip(host).toLowerCase();
+                    if (this.redirectWhiteList.contains(host) || this.redirectWhiteList.contains(KEY_DEFAULT)) {
+                        redirect = true;
+                    } else if (Proxy.Type.DIRECT == proxyType
+                            || (addr != null && StringUtils.equals(uri.getHost(), ((InetSocketAddress) addr).getHostString()))) {
+                        redirect = false;
+                    }
+                }
+                if (redirect) {
+                    result.add(proxy);
+                } else {
+                    result.add(Proxy.NO_PROXY);
+                }
+            }
+        }
+        return result;
+    }
+
+    private URI getNewUri(URI uri, boolean validateUri) {
+        URI newUri = uri;
+        if (newUri != null) {
+            try {
+                // get host before new URI, because the host may be set manually due to URI issue
+                String host = newUri.getHost();
+                newUri = new URI(newUri.getScheme(), newUri.getAuthority(), newUri.getPath(), newUri.getQuery(),
+                        newUri.getFragment());
+                Map<String, String> protocolMap = null;
+                if (StringUtils.isNotBlank(host)) {
+                    protocolMap = hostMap.get(host.toLowerCase());
+                }
+                if (protocolMap == null) {
+                    protocolMap = hostMap.get(KEY_DEFAULT);
+                }
+
+                if (protocolMap != null) {
+                    String schema = newUri.getScheme();
+                    if (schema != null) {
+                        String lowercasedProtocol = schema.toLowerCase();
+                        String preferedProtocol = protocolMap.get(lowercasedProtocol);
+                        if (StringUtils.isNotBlank(preferedProtocol)) {
+                            /**
+                             * Note: MUST use the constructor which requires authority, because some uri may be illegal,
+                             * then host info will be stored in authority field instead of host filed
+                             */
+                            newUri = new URI(preferedProtocol, newUri.getAuthority(), newUri.getPath(), newUri.getQuery(),
+                                    newUri.getFragment());
+                        }
+                    }
+                }
+            } catch (URISyntaxException e) {
+                if (printProxyLog) {
+                    ExceptionHandler.process(
+                            new Exception(Messages.getString("TalendProxySelector.exception.proxySelectionError", uri), e),
+                            Priority.WARN);
+                }
+            }
+        }
+        if (validateUri) {
+            newUri = validateUri(newUri);
+        }
+        return newUri;
+    }
+
+    private Set<Proxy> getProxysFromProviders(final URI uri) {
+        final Set<Proxy> resultFromProviders = new LinkedHashSet<>();
+        Collection<IProxySelectorProvider> providers = getCustomProviders(uri);
         if (providers != null) {
             providers.stream().forEach(p -> {
                 if (instance == p) {
@@ -100,49 +483,126 @@ public class TalendProxySelector extends ProxySelector {
                 }
             });
         }
-        List<Proxy> result = new ArrayList<>();
+        return resultFromProviders;
+    }
 
-        if (resultFromProviders != null && !resultFromProviders.isEmpty()) {
-            result.addAll(resultFromProviders);
-        }
-
-        ProxySelector defaultProxySelector = getDefaultProxySelector();
-        if (defaultProxySelector != null) {
-            List<Proxy> defaultProxys = defaultProxySelector.select(uri);
-            if (defaultProxys != null && !defaultProxys.isEmpty()) {
-                result.addAll(defaultProxys);
+    private Collection<IProxySelectorProvider> getCustomProviders(final URI uri) {
+        Collection<IProxySelectorProvider> providers = Collections.EMPTY_LIST;
+        Collection<Object> possibleKeys = getPossibleKeys(uri);
+        for (Object key : possibleKeys) {
+            providers = this.selectorProviders.get(key);
+            if (providers != null) {
+                break;
             }
         }
-        return result;
+        return providers;
+    }
+
+    public PasswordAuthentication getHttpPasswordAuthentication() {
+        String[] schemas = new String[] { "http", "https" };
+        for (String schema : schemas) {
+            String proxyUser = System.getProperty(schema + ".proxyUser");
+            String proxyPassword = System.getProperty(schema + ".proxyPassword");
+
+            if (StringUtils.isNotBlank(proxyUser)) {
+                char[] pwdChars = new char[0];
+                if (proxyPassword != null && !proxyPassword.isEmpty()) {
+                    pwdChars = proxyPassword.toCharArray();
+                }
+                return new PasswordAuthentication(proxyUser, pwdChars);
+            }
+        }
+        return null;
     }
 
     public boolean addProxySelectorProvider(IProxySelectorProvider provider) {
-        List<IProxySelectorProvider> proxySelectorProviders = getProxySelectorProviders();
-        if (!proxySelectorProviders.contains(provider)) {
-            return proxySelectorProviders.add(provider);
+        try {
+            Object key = provider.getKey();
+            Collection<IProxySelectorProvider> collection = this.selectorProviders.get(key);
+            if (collection == null) {
+                synchronized (this.selectorProviders) {
+                    collection = this.selectorProviders.get(key);
+                    if (collection == null) {
+                        collection = Collections.synchronizedList(new LinkedList<>());
+                        this.selectorProviders.put(key, collection);
+                    }
+                }
+            }
+            collection.add(provider);
+            return true;
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
         }
         return false;
     }
 
     public boolean removeProxySelectorProvider(IProxySelectorProvider provider) {
-        return getProxySelectorProviders().remove(provider);
-    }
-
-    private List<IProxySelectorProvider> getProxySelectorProviders() {
-        return selectorProviders;
+        try {
+            Object key = provider.getKey();
+            Collection<IProxySelectorProvider> collection = this.selectorProviders.get(key);
+            if (collection != null) {
+                synchronized (this.selectorProviders) {
+                    collection = this.selectorProviders.get(key);
+                    if (collection != null) {
+                        collection.remove(provider);
+                        if (collection.isEmpty()) {
+                            this.selectorProviders.remove(key);
+                        }
+                    }
+                }
+            }
+            return true;
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+        return false;
     }
 
     public ProxySelector getDefaultProxySelector() {
-        return defaultSelector;
+        switch (eProxySelector) {
+        case jre:
+            return getJreProxySelector();
+        default:
+            return eclipseDefaultSelector;
+        }
     }
 
-    public void setDefaultProxySelector(final ProxySelector selector) {
-        defaultSelector = selector;
+    private ProxySelector getJreProxySelector() {
+        try {
+            /**
+             * for tcompv0, daikon may create an isolated system properties for it, so proxies may be ignored in the new
+             * system properties; here we try to call the method to add proxies into the isolated system properties
+             */
+            if (updateSystemPropertiesForJre && ClassLoaderIsolatedSystemProperties.getInstance()
+                    .isIsolated(Thread.currentThread().getContextClassLoader())) {
+                if (printProxyLog) {
+                    ExceptionHandler.log("Before update jre proxy system properties for the isolated classloader, http.proxyHost="
+                            + System.getProperty("http.proxyHost"));
+                }
+                IProxyService proxyService = CommonsPlugin.getProxyService();
+                proxyManagerUpdateSystemPropertiesFunc.invoke(proxyService);
+                if (printProxyLog) {
+                    ExceptionHandler.log("After updated jre proxy system properties for the isolated classloader, http.proxyHost="
+                            + System.getProperty("http.proxyHost"));
+                }
+            }
+        } catch (Throwable e) {
+            ExceptionHandler.process(e);
+        }
+        return this.jreDefaultSelector;
+    }
+
+    public ProxySelector getEclipseDefaultSelector() {
+        return eclipseDefaultSelector;
+    }
+
+    public void setEclipseDefaultSelector(ProxySelector eclipseDefaultSelector) {
+        this.eclipseDefaultSelector = eclipseDefaultSelector;
     }
 
     @Override
     public void connectFailed(final URI uri, final SocketAddress sa, final IOException ioe) {
-        List<IProxySelectorProvider> providers = getProxySelectorProviders();
+        Collection<IProxySelectorProvider> providers = getCustomProviders(uri);
         if (providers != null) {
             providers.stream().forEach(p -> {
                 if (p.canHandle(uri)) {
@@ -151,23 +611,35 @@ public class TalendProxySelector extends ProxySelector {
             });
         }
 
-        ProxySelector defaultProxySelector = getDefaultProxySelector();
-        if (defaultProxySelector != null) {
-            defaultProxySelector.connectFailed(uri, sa, ioe);
+        if (executeConnectionFailed) {
+            /**
+             * Just try to make the behavior of jre proxy selector same like eclipse proxy selector
+             */
+            ProxySelector defaultProxySelector = getDefaultProxySelector();
+            if (defaultProxySelector != null) {
+                defaultProxySelector.connectFailed(uri, sa, ioe);
+            }
         }
     }
 
-    public static abstract class AbstractProxySelectorProvider implements IProxySelectorProvider {
-
-        @Override
-        public void connectFailed(final URI uri, final SocketAddress sa, final IOException ioe) {
-            // nothing to do
+    public IProxySelectorProvider createDefaultProxySelectorProvider() {
+        if (disableDefaultSelectorProvider) {
+            return null;
         }
+        return new DefaultProxySelectorProvider(Thread.currentThread());
+    }
 
+    public IProxySelectorProvider createDefaultProxySelectorProvider(String host) {
+        if (disableDefaultSelectorProvider) {
+            return null;
+        }
+        return new DefaultProxySelectorProvider(host);
     }
 
     public static interface IProxySelectorProvider {
 
+        Object getKey();
+
         boolean canHandle(final URI uri);
 
         List<Proxy> select(final URI uri);
@@ -176,4 +648,120 @@ public class TalendProxySelector extends ProxySelector {
 
     }
 
+    public static abstract class AbstractProxySelectorProvider implements IProxySelectorProvider {
+
+        private boolean isDebugMode = CommonsPlugin.isDebugMode();
+
+        @Override
+        public Object getKey() {
+            return this;
+        }
+
+        @Override
+        public void connectFailed(final URI uri, final SocketAddress sa, final IOException ioe) {
+            if (isDebugMode) {
+                ExceptionHandler.process(ioe);
+            }
+        }
+
+    }
+
+    public static Collection<Object> getPossibleKeys(URI uri) {
+        Collection<Object> possibleKeys = new ArrayList<>();
+        possibleKeys.add(Thread.currentThread());
+        if (uri != null) {
+            String uriHost = uri.getHost();
+            if (StringUtils.isNotBlank(uriHost)) {
+                possibleKeys.add(uriHost);
+            }
+        }
+        return possibleKeys;
+    }
+
+    private enum EProxySelector {
+        eclipse_default,
+        jre
+    }
+
+    private class DefaultProxySelectorProvider extends TalendProxySelector.AbstractProxySelectorProvider {
+
+        private Thread currentThread = null;
+
+        private String host = null;
+
+        public DefaultProxySelectorProvider(Thread thread) {
+            this.currentThread = thread;
+        }
+
+        public DefaultProxySelectorProvider(String host) {
+            this.host = host;
+            if (StringUtils.isNotBlank(this.host)) {
+                this.host = this.host.toLowerCase();
+            }
+        }
+
+        @Override
+        public Object getKey() {
+            if (this.currentThread != null) {
+                return currentThread;
+            }
+            if (this.host != null) {
+                return this.host;
+            }
+            return super.getKey();
+        }
+
+        @Override
+        public boolean canHandle(URI uri) {
+            if (disableDefaultSelectorProvider) {
+                return false;
+            }
+            if (currentThread != null && Thread.currentThread() == currentThread) {
+                return true;
+            }
+            if (host != null) {
+                if (uri == null) {
+                    return false;
+                }
+                String uriHost = uri.getHost();
+                if (StringUtils.isNotBlank(uriHost)) {
+                    return this.host.equals(uriHost.toLowerCase());
+                }
+                return false;
+            }
+            return false;
+        }
+
+        @Override
+        public List<Proxy> select(URI uri) {
+            List<Proxy> result = new ArrayList<>();
+            try {
+                ProxySelector defaultProxySelector = getDefaultProxySelector();
+                if (defaultProxySelector != null) {
+                    List<Proxy> defaultProxys = defaultProxySelector.select(uri);
+                    if (defaultProxys != null && !defaultProxys.isEmpty()) {
+                        for (Proxy proxy : defaultProxys) {
+                            SocketAddress addr = null;
+                            Proxy.Type proxyType = null;
+                            if (proxy != null) {
+                                proxyType = proxy.type();
+                                addr = proxy.address();
+                            }
+                            if (Proxy.Type.DIRECT == proxyType || (addr != null
+                                    && StringUtils.equals(uri.getHost(), ((InetSocketAddress) addr).getHostString()))) {
+                                result.add(Proxy.NO_PROXY);
+                            } else {
+                                result.add(proxy);
+                            }
+                        }
+                    }
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+            return result;
+        }
+
+    }
+
 }

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/resource/UpdatesHelper.java

@@ -58,6 +58,8 @@ public class UpdatesHelper {
 
     public static final String COMPONENT_SUFFIX = "_java.xml";
 
+    public static final String COMPONENT_TEMPLATES = "templates";
+
     public static final String NEW_COMPONENT_PREFIX = "installer$$";
 
     public static boolean existArtifacts(File base) {
@@ -296,6 +298,9 @@ public class UpdatesHelper {
 
     public static boolean isOldComponent(File f) {
         if (f != null && f.exists() && f.isDirectory()) {
+            if (f.getName().equals(COMPONENT_TEMPLATES)) {
+                return true;
+            }
             File[] listFiles = f.listFiles();
             if (listFiles != null) {
                 for (File subFile : listFiles) {

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/drawing/background/BackgroundRefresher.java

@@ -339,7 +339,7 @@ public class BackgroundRefresher implements IBackgroundRefresher {
             }
             if ((WindowSystem.isGTK() || EnvironmentUtils.isMacOsSytem()) && child instanceof Tree) {
                 returnedPoint.y += ((Tree) child).getHeaderHeight();
-                returnedPoint.y += ((Table) child).getItemHeight();
+                returnedPoint.y += ((Tree) child).getItemHeight();
             }
             child = child.getParent();
             ScrollBar vScrollBar = child.getVerticalBar();

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/linking/TableToTreeLinker.java

@@ -104,7 +104,6 @@ public class TableToTreeLinker<D1, D2> extends BgDrawableComposite implements IB
         this.source = sourceTable;
 
         dataToTableItemCache = new DataToTableItemCache(sourceTable);
-
     }
 
     protected IStyleLink getDefaultStyleLink() {
@@ -232,7 +231,7 @@ public class TableToTreeLinker<D1, D2> extends BgDrawableComposite implements IB
 
             int yStraight = sourceToCommonPoint.y + treeItemHeight / 2 + tableItemBounds.y;
             pointEndStraight.x = sourceToCommonPoint.x + xStartBezierLink;
-            if (Platform.OS_MACOSX.equals(Platform.getOS())) {
+            if (Platform.OS_MACOSX.equals(Platform.getOS()) || Platform.OS_LINUX.equals(Platform.getOS())) {
                 pointStartStraight.x = sourceToCommonPoint.x + tableItem.getParent().getBounds().width;
                 pointEndStraight.x = pointStartStraight.x;
             } else {
@@ -311,10 +310,10 @@ public class TableToTreeLinker<D1, D2> extends BgDrawableComposite implements IB
                 // Added by Marvin Wang on Nov. 28, 2012 for bug TDI-23378. This is not the best way to fix this issue,
                 // but till now I have not found the root cause.
                 if (Platform.OS_LINUX.equals(Platform.getOS())) {
-                    pointEndCentralCurve.y = pointEndCentralCurve.y - tableItem.getBounds().height;
+                    pointEndCentralCurve.y = pointEndCentralCurve.y - tableItem.getBounds().height - treeItemHeight / 2;
                 }
                 if (Platform.OS_MACOSX.equals(Platform.getOS())) {
-                    pointEndCentralCurve.y = pointEndCentralCurve.y + tableItem.getBounds(0).height;
+                    pointEndCentralCurve.y = pointEndCentralCurve.y - tableItem.getBounds().height;
                 }
                 drawableLink.setPoint1(pointEndStraight);
                 drawableLink.setPoint2(pointEndCentralCurve);

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java

@@ -70,6 +70,7 @@ import org.talend.commons.runtime.service.ITaCoKitService;
 import org.talend.commons.ui.gmf.util.DisplayUtils;
 import org.talend.commons.ui.runtime.exception.MessageBoxExceptionHandler;
 import org.talend.commons.utils.data.container.RootContainer;
+import org.talend.commons.utils.network.TalendProxySelector;
 import org.talend.commons.utils.time.TimeMeasure;
 import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.AbstractDQModelService;
@@ -2020,6 +2021,11 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
 
             TimeMeasure.begin("logOnProject"); //$NON-NLS-1$
             try {
+                /**
+                 * init/check proxy selector, in case default proxy selector is not registed yet
+                 */
+                TalendProxySelector.checkProxy();
+
                 System.getProperties().put("ReadOnlyUser", Boolean.FALSE.toString()); //$NON-NLS-1$
 
                 // remove the auto-build to enhance the build speed and application's use
@@ -2405,9 +2411,14 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
     @Override
     @SuppressWarnings("unchecked")
     public void executeRepositoryWorkUnit(RepositoryWorkUnit workUnit) {
+        checkProxySettings();
         this.repositoryFactoryFromProvider.executeRepositoryWorkUnit(workUnit);
     }
 
+    private void checkProxySettings() {
+        TalendProxySelector.checkProxy();
+    }
+
     @Override
     public void unloadResources(Property property) throws PersistenceException {
         repositoryFactoryFromProvider.unloadResources(property);

main/plugins/org.talend.core.runtime/mappings/mapping_XSD.xml

@@ -29,6 +29,8 @@
 			<dbType type="xs:time" ignoreLen="true" ignorePre="true"/>
 			<dbType type="xs:decimal" ignoreLen="true" ignorePre="true"/>
 			<dbType type="xs:QName" ignoreLen="true" ignorePre="true"/>
+			<dbType type="xs:list" ignoreLen="true" ignorePre="true"/>
+			<dbType type="xs:union" ignoreLen="true" ignorePre="true"/>
 		</dbTypes>
 
 		<language name="java">
@@ -141,6 +143,12 @@
 				<dbType type="xs:time">
 					<talendType type="id_Date" default="true" />
 				</dbType>
+				<dbType type="xs:list">
+					<talendType type="id_List" default="true" />
+				</dbType>
+				<dbType type="xs:union">
+					<talendType type="id_Object" default="true" />
+				</dbType>
 			</dbToTalendTypes>
 		</language>
 	</dbms>

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ILibraryManagerService.java

@@ -206,8 +206,12 @@ public interface ILibraryManagerService extends IService {
      */
     public boolean isJarNeedToBeDeployed(File jarFile);
 
+    public boolean isSameFile(File f1, File f2);
+
     public void checkModuleStatus(ModuleNeeded module);
 
     public String getJarNameFromMavenuri(String mavenURI);
 
+    public void guessMavenRUIFromIndex(File jarFile, Map<String, String> sourceAndMavenUri);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/ConnParameterKeys.java

@@ -302,4 +302,11 @@ public class ConnParameterKeys {
 
     public static final String CONN_PARA_KEY_DATABRICKS_DBFS_DEP_FOLDER="CONN_PARA_KEY_DATABRICKS_DBFS_DEP_FOLDER";
 
+    /**
+     * Override hadoop configuration
+     */
+    public static final String CONN_PARA_KEY_SET_HADOOP_CONF = "CONN_PARA_KEY_SET_HADOOP_CONF";
+
+    public static final String CONN_PARA_KEY_HADOOP_CONF_SPECIFIC_JAR = "CONN_PARA_KEY_HADOOP_CONF_SPECIFIC_JAR";
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/DatabaseConnStrUtil.java

@@ -47,6 +47,8 @@ public class DatabaseConnStrUtil {
     // for match url has :<port> exist
     private static final String PATTERN_PORT = "(:\\d{1,5})";
 
+    private static final String DATABASE_STRING = "DATABASE=";
+
     private static String getStringReplace(final String init, final String before, final String after,
             final boolean supportContext) {
         return getStringReplace(init, before, after, supportContext, false);
@@ -127,6 +129,13 @@ public class DatabaseConnStrUtil {
                 } else {
                     s = getStringReplace(s, EDatabaseConnVar.PORT.getVariable(), port, supportContext);
                 }
+                if (EDatabaseConnTemplate.TERADATA.equals(connStr)) {
+                    if (StringUtils.isNotBlank(sid)) {
+                        s = getStringReplace(s, EDatabaseConnVar.SID.getVariable(), DATABASE_STRING + sid, supportContext);
+                    } else {
+                        s = getStringReplace(s, EDatabaseConnVar.SID.getVariable() + ",", sid, supportContext); //$NON-NLS-1$
+                    }
+                }
                 s = getStringReplace(s, EDatabaseConnVar.SID.getVariable(), sid, supportContext);
                 s = getStringReplace(s, EDatabaseConnVar.SERVICE_NAME.getVariable(), sid, supportContext);
                 s = getStringReplace(s, EDatabaseConnVar.DATASOURCE.getVariable(), datasource, supportContext);
@@ -565,7 +574,7 @@ public class DatabaseConnStrUtil {
         List<ERepositoryObjectType> extraTypes = new ArrayList<>();
         IGenericDBService dbService = null;
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericDBService.class)) {
-            dbService = (IGenericDBService) GlobalServiceRegister.getDefault().getService(IGenericDBService.class);
+            dbService = GlobalServiceRegister.getDefault().getService(IGenericDBService.class);
         }
         if (dbService != null) {
             extraTypes.addAll(dbService.getExtraTypes());

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/version/EDatabaseVersion4Drivers.java

@@ -166,9 +166,9 @@ public enum EDatabaseVersion4Drivers {
     REDSHIFT(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT, "redshift", "REDSHIFT", //$NON-NLS-1$ //$NON-NLS-2$
             "redshift-jdbc42-no-awssdk-1.2.20.1043.jar")), //$NON-NLS-1$
     REDSHIFT_SSO(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT_SSO, "redshift sso", "REDSHIFT_SSO", //$NON-NLS-1$ //$NON-NLS-2$
-            new String[] { "redshift-jdbc42-no-awssdk-1.2.20.1043.jar", "aws-java-sdk-1.11.406.jar", "jackson-core-2.9.5.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-                    "jackson-databind-2.9.5.jar", "jackson-annotations-2.9.0.jar", "httpcore-4.4.9.jar", "httpclient-4.5.5.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$//$NON-NLS-4$
-                    "joda-time-2.8.1.jar", "commons-logging-1.1.3.jar" })), //$NON-NLS-1$ //$NON-NLS-2$
+            new String[] { "redshift-jdbc42-no-awssdk-1.2.20.1043.jar", "aws-java-sdk-1.11.406.jar", "jackson-core-2.9.9.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+                    "jackson-databind-2.9.9.jar", "jackson-annotations-2.9.0.jar", "httpcore-4.4.9.jar", "httpclient-4.5.5.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$//$NON-NLS-4$
+                    "joda-time-2.8.1.jar", "commons-logging-1.1.3.jar", "commons-codec-1.6.jar" })), //$NON-NLS-1$ //$NON-NLS-2$
 
     AMAZON_AURORA(new DbVersion4Drivers(EDatabaseTypeName.AMAZON_AURORA, "mysql-connector-java-5.1.30-bin.jar")); //$NON-NLS-1$
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/HadoopClassLoaderFactory2.java

@@ -12,9 +12,13 @@
 // ============================================================================
 package org.talend.core.hadoop;
 
+import java.io.File;
 import java.net.MalformedURLException;
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Consumer;
 
+import org.apache.commons.lang.StringUtils;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.classloader.ClassLoaderFactory;
 import org.talend.core.classloader.DynamicClassLoader;
@@ -67,9 +71,18 @@ public class HadoopClassLoaderFactory2 {
         return getClassLoader(relatedClusterId, EHadoopCategory.HBASE, distribution, version, useKrb);
     }
 
-    public static ClassLoader getHadoopCustomClassLoader(String uid, Object customJars) {
-        return HadoopClassLoaderFactory2.builder().withTypePrefix(EHadoopCategory.CUSTOM.getName()).withUid(uid)
-                .build(customJars, true);
+    public static ClassLoader getHadoopCustomClassLoader(String uid, String relatedClusterId, EHadoopCategory category,
+            Object customJars, boolean useKrb) {
+        ClassLoader classLoader = HadoopClassLoaderFactory2.builder().withTypePrefix(EHadoopCategory.CUSTOM.getName())
+                .withUid(uid).build(customJars, true);
+        if (StringUtils.isNotBlank(relatedClusterId) && classLoader instanceof DynamicClassLoader) {
+            try {
+                classLoader = addExtraJars(relatedClusterId, category, (DynamicClassLoader) classLoader, useKrb);
+            } catch (MalformedURLException e) {
+                ExceptionHandler.process(e);
+            }
+        }
+        return classLoader;
     }
 
     public static ClassLoader getHCatalogClassLoader(String relatedClusterId, String distribution, String version, boolean useKrb) {
@@ -198,16 +211,35 @@ public class HadoopClassLoaderFactory2 {
             String[] addedJars = null;
             String[] excludedJars = null;
             String[] securityJars = getSecurityJars(category);
-            String customConfsJarName = hadoopClusterService.getCustomConfsJarName(relatedClusterId);
-            if (customConfsJarName != null) {
-                addedJars = new String[] { customConfsJarName };
+
+            Optional<HadoopConfJarBean> customConfsJar = hadoopClusterService.getCustomConfsJar(relatedClusterId);
+            HadoopConfJarBean confJarBean = customConfsJar.orElse(null);
+            String customConfsJarName = customConfsJar.map(b -> b.getCustomConfJarName()).orElse(null);
+            Consumer<DynamicClassLoader> afterLoaded = null;
+            if (confJarBean != null) {
+                if (confJarBean.isOverrideCustomConf()) {
+                    String overrideCustomConfPath = confJarBean.getOriginalOverrideCustomConfPath();
+                    if (StringUtils.isBlank(overrideCustomConfPath) || !new File(overrideCustomConfPath).exists()) {
+                        ExceptionHandler.process(
+                                new Exception("Set Hadoop configuration JAR path is invalid: " + overrideCustomConfPath));
+                    } else {
+                        afterLoaded = (t) -> t.addLibrary(overrideCustomConfPath);
+                    }
+                } else {
+                    if (StringUtils.isNotBlank(customConfsJarName)) {
+                        addedJars = new String[] { customConfsJarName };
+                    }
+                }
                 excludedJars = securityJars;
             } else if (useKrb) {
                 addedJars = securityJars;
                 excludedJars = new String[] { customConfsJarName };
             }
-            if (addedJars != null || excludedJars != null) {
+            if (addedJars != null || excludedJars != null || afterLoaded != null) {
                 cll = DynamicClassLoader.createNewOneBaseLoader(loader, addedJars, excludedJars);
+                if (afterLoaded != null) {
+                    afterLoaded.accept(cll);
+                }
             }
         }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/HadoopConfJarBean.java

@@ -0,0 +1,92 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.hadoop;
+
+/**
+ * DOC cmeng  class global comment. Detailled comment
+ */
+public class HadoopConfJarBean {
+
+    private boolean overrideCustomConf;
+
+    private boolean isContextMode;
+
+    /**
+     * value saved in connection, it may be a context
+     */
+    private String overrideCustomConfPath;
+
+    /**
+     * original value of the path, if connection is in context mode, then this value if the default value of the context
+     */
+    private String originalOverrideCustomConfPath;
+
+    private String customConfJarName;
+
+    public HadoopConfJarBean(boolean isContextMode, boolean overrideCustomConf, String overrideCustomConfPath,
+            String originalOverrideCustomConfPath, String customConfJarName) {
+        this.isContextMode = isContextMode;
+        this.overrideCustomConf = overrideCustomConf;
+        this.overrideCustomConfPath = overrideCustomConfPath;
+        this.originalOverrideCustomConfPath = originalOverrideCustomConfPath;
+        this.customConfJarName = customConfJarName;
+    }
+
+    public boolean isContextMode() {
+        return isContextMode;
+    }
+
+    public void setContextMode(boolean isContextMode) {
+        this.isContextMode = isContextMode;
+    }
+
+    public boolean isOverrideCustomConf() {
+        return overrideCustomConf;
+    }
+
+    public void setOverrideCustomConf(boolean overrideCustomConf) {
+        this.overrideCustomConf = overrideCustomConf;
+    }
+
+    public String getOriginalOverrideCustomConfPath() {
+        return originalOverrideCustomConfPath;
+    }
+
+    public void setOriginalOverrideCustomConfPath(String originalOverrideCustomConfPath) {
+        this.originalOverrideCustomConfPath = originalOverrideCustomConfPath;
+    }
+
+    public String getOverrideCustomConfPath() {
+        return overrideCustomConfPath;
+    }
+
+    public void setOverrideCustomConfPath(String overrideCustomConfPath) {
+        this.overrideCustomConfPath = overrideCustomConfPath;
+    }
+
+    public String getCustomConfJarName() {
+        return customConfJarName;
+    }
+
+    public void setCustomConfJarName(String customConfJarName) {
+        this.customConfJarName = customConfJarName;
+    }
+
+    @Override
+    public String toString() {
+        return "HadoopConfJarBean [overrideCustomConf=" + overrideCustomConf + ", isContextMode=" + isContextMode
+                + ", overrideCustomConfPath=" + overrideCustomConfPath + ", originalOverrideCustomConfPath="
+                + originalOverrideCustomConfPath + ", customConfJarName=" + customConfJarName + "]";
+    }
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/IHadoopClusterService.java

@@ -14,6 +14,7 @@ package org.talend.core.hadoop;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import org.eclipse.core.runtime.IPath;
 import org.talend.commons.exception.BusinessException;
@@ -232,11 +233,12 @@ public interface IHadoopClusterService extends IService {
      */
     public Item getHadoopClusterItemById(String id);
 
-    public String getCustomConfsJarName(String id);
+    public Optional<HadoopConfJarBean> getCustomConfsJar(String id);
 
-    public String getCustomConfsJarName(String id, boolean createJarIfNotExist, boolean addExtraIds);
+    public Optional<HadoopConfJarBean> getCustomConfsJar(String id, boolean createJarIfNotExist, boolean addExtraIds);
 
-    public String getCustomConfsJarName(ConnectionItem connectionItem, boolean createJarIfNotExist, boolean addExtraIds);
+    public Optional<HadoopConfJarBean> getCustomConfsJar(ConnectionItem connectionItem, boolean createJarIfNotExist,
+            boolean addExtraIds);
 
     public void useCustomConfsJarIfNeeded(List<ModuleNeeded> modulesNeeded, String clusterId);
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/components/IComponent.java

@@ -60,6 +60,15 @@ public interface IComponent {
 
     public String getOriginalName();
 
+    /**
+     * Only for component display (palette,search)
+     * 
+     * @return
+     */
+    default public String getDisplayName() {
+        return getName();
+    }
+
     public String getLongName();
 
     public String getOriginalFamilyName();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/routines/RoutineLibraryMananger.java

@@ -13,6 +13,7 @@
 package org.talend.core.model.routines;
 
 import java.io.File;
+import java.io.IOException;
 import java.net.URI;
 import java.net.URL;
 import java.util.ArrayList;
@@ -31,9 +32,12 @@ import org.osgi.framework.Bundle;
 import org.talend.commons.utils.resource.FileExtensions;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.ILibraryManagerService;
-import org.talend.core.ISVNProviderServiceInCoreRuntime;
-import org.talend.core.PluginChecker;
 import org.talend.core.model.general.LibraryInfo;
+import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.nexus.TalendLibsServerManager;
+import org.talend.core.runtime.maven.MavenArtifact;
+import org.talend.core.runtime.maven.MavenConstants;
+import org.talend.core.runtime.maven.MavenUrlHelper;
 
 /**
  * wchen class global comment. Detailled comment
@@ -96,9 +100,12 @@ public class RoutineLibraryMananger {
                                 try {
                                     URL fileUrl = FileLocator.toFileURL(entry);
                                     if(fileUrl != null){
-                                    	if (!"file".equals(fileUrl.getProtocol())) throw new IllegalArgumentException();
-                                    	URI fileUri = new File(fileUrl.getFile()).toURI();
-                                        libManagerService.deploy(fileUri);
+                                        if (!"file".equals(fileUrl.getProtocol())) throw new IllegalArgumentException();
+                                        File file = new File(fileUrl.getFile());
+                                        if (needDeploy(fileUrl)) {
+                                            URI fileUri = file.toURI();
+                                            libManagerService.deploy(fileUri);
+                                        }
                                     }
                                 } catch (Exception e) {
                                     log.warn("Cannot deploy: " + bundleName + path);
@@ -114,6 +121,56 @@ public class RoutineLibraryMananger {
         }
     }
 
+    private boolean needDeploy(URL fileUrl) throws IOException, Exception {
+        File file = new File(fileUrl.getFile());
+        ILibraryManagerService libManagerService = null;
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibraryManagerService.class)) {
+            libManagerService = (ILibraryManagerService) GlobalServiceRegister.getDefault()
+                    .getService(ILibraryManagerService.class);
+        }
+        if (libManagerService != null) {
+            Map<String, String> sourceAndMavenUri = new HashMap<>();
+            libManagerService.guessMavenRUIFromIndex(file, sourceAndMavenUri);
+            String mavUri = null;
+            boolean isSnapshot = false;
+            for (String key : sourceAndMavenUri.keySet()) {
+                if (sourceAndMavenUri.get(key).equals(file.getPath())) {
+                    mavUri = key;
+                    break;
+                }
+            }
+            if (mavUri == null) {
+                return true;
+            }
+            final MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(mavUri);
+            if (parseMvnUrl != null) {
+                if (parseMvnUrl.getVersion() != null && parseMvnUrl.getVersion().endsWith(MavenConstants.SNAPSHOT)) {
+                    isSnapshot = true;
+                }
+            }
+            TalendLibsServerManager manager = TalendLibsServerManager.getInstance();
+            ArtifactRepositoryBean customNexusServer = manager.getCustomNexusServer();
+            File jarFile = null;
+            try {
+                jarFile = libManagerService.resolveJar(customNexusServer, mavUri);
+            } catch (Exception ex) {
+                // Ignore here
+            }
+            boolean exist = (jarFile != null && jarFile.exists());
+            if (exist) {
+                if (isSnapshot) {
+                    boolean isSame = libManagerService.isSameFile(jarFile, file);
+                    if (!isSame) {
+                        return true;
+                    }
+                }
+            } else {
+                return true;
+            }
+        }
+        return false;
+    }
+
     public Map<String, List<LibraryInfo>> getRoutineAndJars() {
         if (routineAndJars == null) {
             routineAndJars = new HashMap<String, List<LibraryInfo>>();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/ParameterValueUtil.java

@@ -77,21 +77,24 @@ public final class ParameterValueUtil {
                 }
             }
         } else if (param.getValue() instanceof List) { // for TABLE
-            List<Map<String, Object>> tableValues = (List<Map<String, Object>>) param.getValue();
-            for (Map<String, Object> line : tableValues) {
-                for (String key : line.keySet()) {
-                    Object cellValue = line.get(key);
-                    if (cellValue instanceof String) { // cell is text so
-                        // rename data if
-                        // needed
-                        String value = (String) cellValue;
-                        if (value.contains(oldName)) {
-                            // line.put(key, value.replaceAll(oldName,
-                            // newName));
-                            // String newValue = renameValues(value, oldName, newName, flag);
-                            String newValue = splitQueryData(oldName, newName, value);
-                            if (!value.equals(newValue)) {
-                                line.put(key, newValue);
+            List tableValues = (List) param.getValue();
+            for (Object current : tableValues) {
+                if (current != null && current instanceof Map) {
+                    Map<String, Object> line = (Map<String, Object>) current;
+                    for (String key : line.keySet()) {
+                        Object cellValue = line.get(key);
+                        if (cellValue instanceof String) { // cell is text so
+                            // rename data if
+                            // needed
+                            String value = (String) cellValue;
+                            if (value.contains(oldName)) {
+                                // line.put(key, value.replaceAll(oldName,
+                                // newName));
+                                // String newValue = renameValues(value, oldName, newName, flag);
+                                String newValue = splitQueryData(oldName, newName, value);
+                                if (!value.equals(newValue)) {
+                                    line.put(key, newValue);
+                                }
                             }
                         }
                     }
@@ -721,13 +724,22 @@ public final class ParameterValueUtil {
                 return true;
             }
         } else if (param.getValue() instanceof List) { // for TABLE
-            List<Map<String, Object>> tableValues = (List<Map<String, Object>>) param.getValue();
-            for (Map<String, Object> line : tableValues) {
-                for (String key : line.keySet()) {
-                    Object cellValue = line.get(key);
-                    if (cellValue instanceof String) { // cell is text so
-                        // test data
-                        if (ParameterValueUtil.valueContains((String) cellValue, name)) {
+            List tableValues = (List) param.getValue();
+            for (Object current : tableValues) {
+                if (current != null) {
+                    if (current instanceof Map) {
+                        Map<String, Object> line = (Map<String, Object>) current;
+                        for (String key : line.keySet()) {
+                            Object cellValue = line.get(key);
+                            if (cellValue instanceof String) { // cell is text so
+                                // test data
+                                if (ParameterValueUtil.valueContains((String) cellValue, name)) {
+                                    return true;
+                                }
+                            }
+                        }
+                    } else if (current instanceof String) {
+                        if (ParameterValueUtil.valueContains((String) current, name)) {
                             return true;
                         }
                     }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/HttpClientTransport.java

@@ -12,14 +12,12 @@
 // ============================================================================
 package org.talend.core.nexus;
 
-import java.net.Authenticator;
 import java.net.InetSocketAddress;
 import java.net.PasswordAuthentication;
 import java.net.Proxy;
 import java.net.Proxy.Type;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
@@ -37,7 +35,6 @@ import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.OperationCanceledException;
 import org.talend.commons.exception.BusinessException;
-import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.utils.network.TalendProxySelector;
 import org.talend.commons.utils.network.TalendProxySelector.IProxySelectorProvider;
 import org.talend.core.runtime.CoreRuntimePlugin;
@@ -167,7 +164,7 @@ public abstract class HttpClientTransport {
         }
     }
 
-    private IProxySelectorProvider addProxy(final DefaultHttpClient httpClient, URI requestURI) {
+    public static IProxySelectorProvider addProxy(final DefaultHttpClient httpClient, URI requestURI) {
         IProxySelectorProvider proxySelectorProvider = null;
         try {
             if (Boolean.valueOf(System.getProperty(PROP_PROXY_HTTP_CLIENT_USE_DEFAULT_SETTINGS, Boolean.FALSE.toString()))) {
@@ -185,10 +182,10 @@ public abstract class HttpClientTransport {
                 }
                 final Proxy finalProxy = usedProxy;
                 InetSocketAddress address = (InetSocketAddress) finalProxy.address();
-                String proxyServer = address.getHostName();
+                String proxyServer = address.getHostString();
                 int proxyPort = address.getPort();
-                PasswordAuthentication proxyAuthentication = Authenticator.requestPasswordAuthentication(proxyServer,
-                        address.getAddress(), proxyPort, "Http Proxy", "Http proxy authentication", null);
+                TalendProxySelector proxySelector = TalendProxySelector.getInstance();
+                PasswordAuthentication proxyAuthentication = proxySelector.getHttpPasswordAuthentication();
                 if (proxyAuthentication != null) {
                     String proxyUser = proxyAuthentication.getUserName();
                     if(StringUtils.isNotBlank(proxyUser)){
@@ -203,7 +200,7 @@ public abstract class HttpClientTransport {
                 }
                 HttpHost proxyHost = new HttpHost(proxyServer, proxyPort);
                 httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost);
-                proxySelectorProvider = createProxySelectorProvider();
+                proxySelectorProvider = proxySelector.createDefaultProxySelectorProvider();
             }
             return proxySelectorProvider;
         } finally {
@@ -213,61 +210,12 @@ public abstract class HttpClientTransport {
         }
     }
 
-    private void removeProxy(IProxySelectorProvider proxySelectorProvider) {
+    public static void removeProxy(IProxySelectorProvider proxySelectorProvider) {
         if (proxySelectorProvider != null) {
             TalendProxySelector.getInstance().removeProxySelectorProvider(proxySelectorProvider);
         }
     }
 
-    private IProxySelectorProvider createProxySelectorProvider() {
-        IProxySelectorProvider proxySelectorProvider = new TalendProxySelector.AbstractProxySelectorProvider() {
-
-            private Thread currentThread = Thread.currentThread();
-
-            @Override
-            public List<Proxy> select(URI uri) {
-                // return Collections.EMPTY_LIST;
-
-                List<Proxy> newProxys = new ArrayList<>();
-                if (uri == null) {
-                    return newProxys;
-                }
-                String schema = uri.getScheme();
-                if (schema != null && schema.toLowerCase().startsWith("socket")) { //$NON-NLS-1$
-                    try {
-                        URI newUri = new URI("https", uri.getUserInfo(), uri.getHost(), uri.getPort(), uri.getPath(),
-                                uri.getQuery(), uri.getFragment());
-                        List<Proxy> proxys = TalendProxySelector.getInstance().getDefaultProxySelector().select(newUri);
-                        if (proxys != null && !proxys.isEmpty()) {
-                            newProxys.addAll(proxys);
-                        } else {
-                            newUri = new URI("http", uri.getUserInfo(), uri.getHost(), uri.getPort(), uri.getPath(),
-                                    uri.getQuery(), uri.getFragment());
-                            proxys = TalendProxySelector.getInstance().getDefaultProxySelector().select(newUri);
-                            if (proxys != null && !proxys.isEmpty()) {
-                                newProxys.addAll(proxys);
-                            }
-                        }
-                    } catch (URISyntaxException e) {
-                        ExceptionHandler.process(e);
-                    }
-                }
-                return newProxys;
-
-            }
-
-            @Override
-            public boolean canHandle(URI uri) {
-                if (Thread.currentThread() == currentThread) {
-                    return true;
-                }
-                return false;
-            }
-
-        };
-        return proxySelectorProvider;
-    }
-
     public void processResponseCode(HttpResponse response) throws BusinessException {
         StatusLine statusLine = response.getStatusLine();
         int responseCode = statusLine.getStatusCode();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/TalendMavenResolver.java

@@ -24,6 +24,9 @@ import org.osgi.framework.ServiceReference;
 import org.osgi.service.cm.ManagedService;
 import org.osgi.util.tracker.ServiceTracker;
 import org.osgi.util.tracker.ServiceTrackerCustomizer;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.TalendProxySelector;
+import org.talend.commons.utils.network.TalendProxySelector.IProxySelectorProvider;
 import org.talend.core.runtime.CoreRuntimePlugin;
 
 /**
@@ -102,7 +105,24 @@ public class TalendMavenResolver {
     }
 
     public static File resolve(String mvnUri) throws IOException, RuntimeException {
-        return getMavenResolver().resolve(mvnUri);
+        TalendProxySelector selectorInstance = null;
+        IProxySelectorProvider proxySelector = null;
+        try {
+            try {
+                selectorInstance = TalendProxySelector.getInstance();
+                proxySelector = selectorInstance.createDefaultProxySelectorProvider();
+                if (proxySelector != null) {
+                    selectorInstance.addProxySelectorProvider(proxySelector);
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+            return getMavenResolver().resolve(mvnUri);
+        } finally {
+            if (proxySelector != null && selectorInstance != null) {
+                selectorInstance.removeProxySelectorProvider(proxySelector);
+            }
+        }
     }
 
     public static void upload(String groupId, String artifactId, String classifier, String extension, String version,

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenArtifact.java

@@ -24,7 +24,7 @@ public class MavenArtifact implements Cloneable {
     private static final char ARTIFACT_SEPARATOR = '-';
 
     private String repositoryUrl, groupId, artifactId, version, type, classifier, description, url, license, licenseUrl,
-            distribution, username, password, lastUpdated;
+            distribution, username, password, lastUpdated, sha1, md5;
 
     public String getLastUpdated() {
         return this.lastUpdated;
@@ -138,6 +138,22 @@ public class MavenArtifact implements Cloneable {
         this.password = password;
     }
 
+    public String getSha1() {
+        return sha1;
+    }
+
+    public void setSha1(String sha1) {
+        this.sha1 = sha1;
+    }
+
+    public String getMd5() {
+        return md5;
+    }
+
+    public void setMd5(String md5) {
+        this.md5 = md5;
+    }
+
     /**
      *
      * DOC ggu Comment method "getFileName".

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/AbstractRoutineSynchronizer.java

@@ -19,7 +19,9 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
@@ -79,6 +81,8 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
     }
 
     private Collection<RoutineItem> getAll(ERepositoryObjectType type, boolean syncRef) throws SystemException {
+        // init code project
+        getRunProcessService().getTalendCodeJavaProject(type);
         // remove routine with same name in reference project
         final Map<String, RoutineItem> beansList = new HashMap<String, RoutineItem>();
         for (IRepositoryViewObject obj : getRepositoryService().getProxyRepositoryFactory().getAll(type)) {
@@ -95,27 +99,37 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
         return (IRepositoryService) GlobalServiceRegister.getDefault().getService(IRepositoryService.class);
     }
 
-    private void getReferencedProjectRoutine(final Map<String, RoutineItem> beansList, final Project project,
-            ERepositoryObjectType routineType, boolean syncRef) throws SystemException {
-        for (IRepositoryViewObject obj : getRepositoryService().getProxyRepositoryFactory().getAll(project, routineType)) {
+    private Set<IRepositoryViewObject> getReferencedProjectRoutine(final Map<String, RoutineItem> beansList,
+            final Project project, ERepositoryObjectType routineType, boolean syncRef) throws SystemException {
+        // init ref code project
+        if (syncRef) {
+            getRunProcessService().getTalendCodeJavaProject(routineType, project.getTechnicalLabel());
+        }
+        Set<IRepositoryViewObject> routines = new HashSet<>();
+        routines.addAll(getRepositoryService().getProxyRepositoryFactory().getAll(project, routineType));
+        for (IRepositoryViewObject obj : routines) {
             final String key = obj.getProperty().getLabel();
             // it does not have a routine with same name
             if (!beansList.containsKey(key)) {
                 beansList.put(key, (RoutineItem) obj.getProperty().getItem());
             }
-            if (syncRef) {
-                // sync routine
-                syncRoutine((RoutineItem) obj.getProperty().getItem(), false, true, true);
-            }
+        }
+        for (ProjectReference projectReference : project.getProjectReferenceList()) {
+            routines.addAll(getReferencedProjectRoutine(beansList, new Project(projectReference.getReferencedProject()),
+                    routineType, syncRef));
         }
         if (syncRef) {
+            routines.stream().forEach(obj -> {
+                try {
+                    syncRoutine((RoutineItem) obj.getProperty().getItem(), project.getTechnicalLabel(), true, true);
+                } catch (SystemException e) {
+                    ExceptionHandler.process(e);
+                }
+            });
             // sync system routine
             syncSystemRoutine(project);
         }
-
-        for (ProjectReference projectReference : project.getProjectReferenceList()) {
-            getReferencedProjectRoutine(beansList, new Project(projectReference.getReferencedProject()), routineType, syncRef);
-        }
+        return routines;
     }
 
     protected void syncSystemRoutine(Project project) throws SystemException {
@@ -128,24 +142,17 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
     }
 
     protected IFile getRoutineFile(RoutineItem routineItem) throws SystemException {
-        return getRoutineFile(routineItem, true);
+        return getRoutineFile(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel());
     }
 
-    protected IFile getRoutineFile(RoutineItem routineItem, boolean currentProject) throws SystemException {
-        String projectTechName;
-        if (currentProject) {
-            projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
-        } else {
-            projectTechName = ProjectManager.getInstance().getProject(routineItem).getTechnicalLabel();
-        }
+    protected IFile getRoutineFile(RoutineItem routineItem, String projectTechName) throws SystemException {
         ITalendProcessJavaProject talendProcessJavaProject = getRunProcessService()
                 .getTalendCodeJavaProject(ERepositoryObjectType.getItemType(routineItem), projectTechName);
         if (talendProcessJavaProject == null) {
             return null;
         }
         IFolder routineFolder = talendProcessJavaProject.getSrcSubFolder(null, routineItem.getPackageType());
-        IFile file = routineFolder.getFile(routineItem.getProperty().getLabel() + JavaUtils.JAVA_EXTENSION);
-        return file;
+        return routineFolder.getFile(routineItem.getProperty().getLabel() + JavaUtils.JAVA_EXTENSION);
     }
 
     private IFile getProcessFile(ProcessItem item) throws SystemException {
@@ -193,30 +200,31 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
 
     @Override
     public void syncRoutine(RoutineItem routineItem, boolean copyToTemp) throws SystemException {
-        syncRoutine(routineItem, true, copyToTemp, false);
+        syncRoutine(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), copyToTemp, false);
     }
 
     @Override
     public void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceUpdate) throws SystemException {
-        syncRoutine(routineItem, true, copyToTemp, forceUpdate);
+        syncRoutine(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), copyToTemp, forceUpdate);
     }
 
     @Override
-    public void syncRoutine(RoutineItem routineItem, boolean currentProject, boolean copyToTemp, boolean forceUpdate) throws SystemException {
+    public void syncRoutine(RoutineItem routineItem, String projectTechName, boolean copyToTemp, boolean forceUpdate)
+            throws SystemException {
         boolean needSync = false;
         if (routineItem != null) {
             if (forceUpdate || !isRoutineUptodate(routineItem)) {
                 needSync = true;
             } else {
-                IFile file = getRoutineFile(routineItem, currentProject);
+                IFile file = getRoutineFile(routineItem, projectTechName);
                 if (file != null && !file.exists()) {
                     needSync = true;
                 }
             }
         }
         if (needSync) {
-            doSyncRoutine(routineItem, currentProject, copyToTemp);
-            if (currentProject) {
+            doSyncRoutine(routineItem, projectTechName, copyToTemp);
+            if (ProjectManager.getInstance().getCurrentProject().getTechnicalLabel().equals(projectTechName)) {
                 setRoutineAsUptodate(routineItem);
             }
         }
@@ -224,14 +232,14 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
 
     public void syncRoutine(RoutineItem routineItem) throws SystemException {
         if (routineItem != null) {
-            doSyncRoutine(routineItem, true, true);
+            doSyncRoutine(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), true);
             setRoutineAsUptodate(routineItem);
         }
     }
 
-    private void doSyncRoutine(RoutineItem routineItem, boolean currentProject, boolean copyToTemp) throws SystemException {
+    private void doSyncRoutine(RoutineItem routineItem, String projectTechName, boolean copyToTemp) throws SystemException {
         try {
-            IFile file = getRoutineFile(routineItem, currentProject);
+            IFile file = getRoutineFile(routineItem, projectTechName);
             if (file == null) {
                 return;
             }
@@ -374,7 +382,7 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
     @Override
     public void syncAllBeansForLogOn() throws SystemException {
         for (RoutineItem beanItem : getBeans(true)) {
-            syncRoutine(beanItem, true, true, true);
+            syncRoutine(beanItem, true, true);
         }
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/ITalendSynchronizer.java

@@ -45,7 +45,8 @@ public interface ITalendSynchronizer {
 
     void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceUpdate) throws SystemException;
 
-    void syncRoutine(RoutineItem routineItem, boolean currentProject, boolean copyToTemp, boolean forceUpdate) throws SystemException;
+    void syncRoutine(RoutineItem routineItem, String projectTechName, boolean copyToTemp, boolean forceUpdate)
+            throws SystemException;
 
     IFile getFile(Item item) throws SystemException;
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/core/IDesignerCoreService.java

@@ -195,4 +195,7 @@ public interface IDesignerCoreService extends IService {
     public void setTACReadTimeout(int timeout);
 
     boolean isDelegateNode(INode node);
+
+    boolean isNeedContextInJar(IProcess process);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java

@@ -216,6 +216,8 @@ public interface IRunProcessService extends IService {
 
     ITalendProcessJavaProject getTalendJobJavaProject(Property property);
 
+    IFolder getCodeSrcFolder(ERepositoryObjectType type, String projectTechName);
+
     ITalendProcessJavaProject getTempJavaProject();
 
     void clearProjectRelatedSettings();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/ProjectManager.java

@@ -200,6 +200,10 @@ public final class ProjectManager {
      * return all the referenced projects of current project.
      */
     public List<Project> getAllReferencedProjects(boolean force) {
+        return getAllReferencedProjects(getCurrentProject(), force);
+    }
+
+    public List<Project> getAllReferencedProjects(Project targetProject, boolean force) {
         List<Project> allReferencedprojects = new ArrayList<Project>();
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IProxyRepositoryService.class)) {
             if (this.getCurrentProject() == null) {
@@ -212,7 +216,7 @@ public final class ProjectManager {
             IProxyRepositoryFactory factory = service.getProxyRepositoryFactory();
             if (factory != null) {
                 List<org.talend.core.model.properties.Project> rProjects = factory
-                        .getReferencedProjects(this.getCurrentProject());
+                        .getReferencedProjects(targetProject);
                 if (rProjects != null) {
                     for (org.talend.core.model.properties.Project p : rProjects) {
                         Project project = new Project(p);

main/plugins/org.talend.core/src/main/java/org/talend/core/CoreService.java

@@ -54,6 +54,7 @@ import org.talend.core.model.metadata.QueryUtil;
 import org.talend.core.model.metadata.builder.ConvertionHelper;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.process.ElementParameterParser;
+import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.model.repository.IRepositoryViewObject;
@@ -280,9 +281,10 @@ public class CoreService implements ICoreService {
             ICodeGeneratorService codeGenService = (ICodeGeneratorService) GlobalServiceRegister.getDefault().getService(
                     ICodeGeneratorService.class);
             codeGenService.createRoutineSynchronizer().syncAllRoutinesForLogOn();
-            codeGenService.createRoutineSynchronizer().syncAllPigudfForLogOn();
+            if (ProcessUtils.isRequiredPigUDFs(null)) {
+                codeGenService.createRoutineSynchronizer().syncAllPigudfForLogOn();
+            }
         }
-
     }
 
     @Override

main/plugins/org.talend.core/src/main/java/org/talend/core/model/components/conversions/RenameComponentConversion.java

@@ -43,7 +43,7 @@ public class RenameComponentConversion implements IComponentConversion {
         replaceAllInAllNodesParameterValue(item, oldNodeUniqueName, newNodeUniqueName);
     }
 
-    private static void replaceAllInAllNodesParameterValue(ProcessType item, String oldName, String newName) {
+    protected static void replaceAllInAllNodesParameterValue(ProcessType item, String oldName, String newName) {
         for (Object o : item.getNode()) {
             NodeType nt = (NodeType) o;
             ComponentUtilities.replaceInNodeParameterValue(nt, oldName, newName);

main/plugins/org.talend.core/src/main/java/org/talend/core/model/components/conversions/RenameETLJDBCComponentConversion.java

@@ -0,0 +1,40 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.components.conversions;
+
+import org.talend.core.model.components.ComponentUtilities;
+import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
+import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
+
+public class RenameETLJDBCComponentConversion extends RenameComponentConversion{
+
+    private String newName;
+    
+    private String oldName;
+
+    public RenameETLJDBCComponentConversion(String newName, String oldName) {
+        super(newName);
+        this.newName = newName;
+        this.oldName = oldName;
+    }
+
+    public void transform(NodeType node) {
+        node.setComponentName(newName);
+        ProcessType item = (ProcessType) node.eContainer();
+        String oldNodeUniqueName = ComponentUtilities.getNodeUniqueName(node);
+        String newNodeUniqueName = oldNodeUniqueName.replaceAll(oldName, newName);
+        ComponentUtilities.setNodeUniqueName(node, newNodeUniqueName);
+        replaceAllInAllNodesParameterValue(item, oldNodeUniqueName, newNodeUniqueName);
+    }
+
+}

main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java

@@ -64,6 +64,8 @@ import org.talend.core.language.LanguageManager;
 import org.talend.core.model.components.ComponentCategory;
 import org.talend.core.model.components.EComponentType;
 import org.talend.core.model.components.IComponent;
+import org.talend.core.model.components.IComponentsFactory;
+import org.talend.core.model.components.IComponentsService;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.Project;
 import org.talend.core.model.metadata.IMetadataColumn;
@@ -81,6 +83,7 @@ import org.talend.core.model.process.JobInfo;
 import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.process.ReplaceNodesInProcessProvider;
 import org.talend.core.model.properties.Item;
+import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.relationship.Relation;
@@ -170,6 +173,8 @@ public class ProcessorUtilities {
 
     private static boolean isDebug = false;
 
+    private static boolean isCIMode = false;
+
     private static JobInfo mainJobInfo;
 
     public static void addOpenEditor(IEditorPart editor) {
@@ -613,28 +618,55 @@ public class ProcessorUtilities {
         return processor;
     }
 
-    private static boolean checkLoopDependencies(Relation mainJobInfo, Map<String, String> idToLastestVersionMap)
+    public static boolean checkLoopDependencies(Relation mainJobInfo, Map<String, String> idToLastestVersionMap)
             throws ProcessorException {
         List<Relation> itemsJobRelatedTo = getItemsRelation(mainJobInfo, idToLastestVersionMap);
         List<Relation> relationChecked = new ArrayList<>();
         relationChecked.add(mainJobInfo);
-        return checkLoopDependencies(mainJobInfo, itemsJobRelatedTo, relationChecked, idToLastestVersionMap);
+        return checkLoopDependencies(mainJobInfo, mainJobInfo, itemsJobRelatedTo, relationChecked, idToLastestVersionMap);
     }
 
-    private static boolean checkLoopDependencies(Relation mainRelation, List<Relation> itemsJobRelatedTo,
+    private static boolean checkLoopDependencies(Relation mainRelation, Relation currentRelation,
+            List<Relation> itemsJobRelatedTo,
             List<Relation> relationChecked, Map<String, String> idToLastestVersionMap) throws ProcessorException {
         boolean hasDependency = false;
         for (Relation relation : itemsJobRelatedTo) {
+            try {
+                // means the tRunjob deactivate, or one of the specific version tRunjon deactivate, skip
+                Map<String, Set<String>> actTrunjobHM = getActivateTRunjobMap(currentRelation.getId(),
+                        currentRelation.getVersion());
+                if (actTrunjobHM.get(relation.getId()) == null
+                        || !actTrunjobHM.get(relation.getId()).contains(relation.getVersion())) {
+                    continue;
+                }
+            } catch (Exception e) {
+                throw new ProcessorException(e);
+            }
+
             hasDependency = relation.getId().equals(mainRelation.getId())
                     && relation.getVersion().equals(mainRelation.getVersion());
             if (!hasDependency) {
                 List<Relation> itemsChildJob = getItemsRelation(relation, idToLastestVersionMap);
                 if (!relationChecked.contains(relation)) {
                     relationChecked.add(relation);
-                    hasDependency = checkLoopDependencies(mainRelation, itemsChildJob, relationChecked, idToLastestVersionMap);
+                    hasDependency = checkLoopDependencies(mainRelation, relation, itemsChildJob, relationChecked,
+                            idToLastestVersionMap);
                 }
                 if (!hasDependency) {
                     for (Relation childRelation : itemsChildJob) {
+
+                        try {
+                            // means the tRunjob deactivate, or one of the specific version tRunjon deactivate, skip
+                            Map<String, Set<String>> activateTRunjobMap = getActivateTRunjobMap(relation.getId(),
+                                    relation.getVersion());
+                            if (activateTRunjobMap.get(childRelation.getId()) == null
+                                    || !activateTRunjobMap.get(childRelation.getId()).contains(childRelation.getVersion())) {
+                                continue;
+                            }
+                        } catch (Exception e) {
+                            throw new ProcessorException(e);
+                        }
+
                         hasDependency = checkLoopDependencies(childRelation, idToLastestVersionMap);
                         if (hasDependency) {
                             break;
@@ -650,6 +682,133 @@ public class ProcessorUtilities {
         return hasDependency;
     }
 
+    private static Map<String, Set<String>> getActivateTRunjobMap(String id, String version) throws PersistenceException {
+        Map<String, Set<String>> actTrunjobHM = new HashMap<String, Set<String>>();
+        ProcessType processType = null;
+        try {
+            IRepositoryViewObject currentJobObject = ProxyRepositoryFactory.getInstance().getSpecificVersion(id, version, true);
+            if (currentJobObject != null) {
+                Item item = currentJobObject.getProperty().getItem();
+                if (item instanceof ProcessItem) {
+                    processType = ((ProcessItem) item).getProcess();
+                } else if (item instanceof JobletProcessItem) {
+                    processType = ((JobletProcessItem) item).getJobletProcess();
+                }
+            }
+        } catch (PersistenceException e) {
+            ExceptionHandler.process(e);
+        }
+        if (processType != null) {
+            List<Project> allProjects = new ArrayList<Project>();
+            allProjects.add(ProjectManager.getInstance().getCurrentProject());
+            allProjects.addAll(ProjectManager.getInstance().getAllReferencedProjects());
+
+            List<String> jobletsComponentsList = new ArrayList<String>();
+            IComponentsFactory componentsFactory = null;
+            if (GlobalServiceRegister.getDefault().isServiceRegistered(IComponentsService.class)) {
+                IComponentsService compService = (IComponentsService) GlobalServiceRegister.getDefault()
+                        .getService(IComponentsService.class);
+                if (compService != null) {
+                    componentsFactory = compService.getComponentsFactory();
+                    for (IComponent component : componentsFactory.readComponents()) {
+                        if (component.getComponentType() == EComponentType.JOBLET) {
+                            jobletsComponentsList.add(component.getName());
+                        }
+                    }
+                }
+            }
+
+            String jobletPaletteType = null;
+            String frameWork = processType.getFramework();
+            if (StringUtils.isBlank(frameWork)) {
+                jobletPaletteType = ComponentCategory.CATEGORY_4_DI.getName();
+            } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK)) {
+                jobletPaletteType = ComponentCategory.CATEGORY_4_SPARK.getName();
+            } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK_STREAMING)) {
+                jobletPaletteType = ComponentCategory.CATEGORY_4_SPARKSTREAMING.getName();
+            }
+
+            for (Object nodeObject : processType.getNode()) {
+                NodeType node = (NodeType) nodeObject;
+                // not tRunjob && not joblet then continue
+                if (!node.getComponentName().equals("tRunJob") && !jobletsComponentsList.contains(node.getComponentName())) { // $NON-NLS-1$
+                    continue;
+                }
+                boolean nodeActivate = true;
+                String processIds = null;
+                String processVersion = null;
+                for (Object elementParam : node.getElementParameter()) {
+                    ElementParameterType elemParamType = (ElementParameterType) elementParam;
+                    if ("PROCESS:PROCESS_TYPE_PROCESS".equals(elemParamType.getName())) { // $NON-NLS-1$
+                        processIds = elemParamType.getValue();
+                        if (StringUtils.isNotBlank(processIds)) {
+                            for (String jobId : processIds.split(ProcessorUtilities.COMMA)) {
+                                if (actTrunjobHM.get(jobId) == null) {
+                                    actTrunjobHM.put(jobId, new HashSet<String>());
+                                }
+                            }
+                        }
+                    } else if ("PROCESS:PROCESS_TYPE_VERSION".equals(elemParamType.getName()) // $NON-NLS-1$
+                            || "PROCESS_TYPE_VERSION".equals(elemParamType.getName())) { // $NON-NLS-1$
+                        processVersion = elemParamType.getValue();
+                    } else if ("ACTIVATE".equals(elemParamType.getName())) { // $NON-NLS-1$
+                        nodeActivate = Boolean.parseBoolean(elemParamType.getValue());
+                    }
+                }
+
+                if (nodeActivate) {
+                    if (StringUtils.isNotBlank(processIds)) {
+                        for (String jobId : processIds.split(ProcessorUtilities.COMMA)) {
+                            String actualVersion = processVersion;
+                            if (RelationshipItemBuilder.LATEST_VERSION.equals(processVersion)) {
+                                for (Project project : allProjects) {
+                                    IRepositoryViewObject lastVersion = null;
+                                    lastVersion = ProxyRepositoryFactory.getInstance().getLastVersion(project, jobId);
+                                    if (lastVersion != null) {
+                                        actualVersion = lastVersion.getVersion();
+                                        break;
+                                    }
+                                }
+                            }
+                            if (actTrunjobHM.get(jobId) != null) {
+                                actTrunjobHM.get(jobId).add(actualVersion);
+                            }
+
+                        }
+                    } else if (componentsFactory != null && jobletPaletteType != null) {
+                        // for joblet
+                        IComponent cc = componentsFactory.get(node.getComponentName(), jobletPaletteType);
+                        if (GlobalServiceRegister.getDefault().isServiceRegistered(IJobletProviderService.class)) {
+                            IJobletProviderService jobletService = (IJobletProviderService) GlobalServiceRegister.getDefault()
+                                    .getService(IJobletProviderService.class);
+                            Property property = jobletService.getJobletComponentItem(cc);
+                            if (property != null && StringUtils.isNotBlank(property.getId())) {
+                                String jobletId = property.getId();
+                                if (actTrunjobHM.get(jobletId) == null) {
+                                    actTrunjobHM.put(jobletId, new HashSet<String>());
+                                }
+                                String actualVersion = processVersion;
+                                if (RelationshipItemBuilder.LATEST_VERSION.equals(processVersion)) {
+                                    for (Project project : allProjects) {
+                                        IRepositoryViewObject lastVersion = null;
+                                        lastVersion = ProxyRepositoryFactory.getInstance().getLastVersion(project, jobletId);
+                                        if (lastVersion != null) {
+                                            actualVersion = lastVersion.getVersion();
+                                            break;
+                                        }
+                                    }
+                                }
+
+                                actTrunjobHM.get(jobletId).add(actualVersion);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return actTrunjobHM;
+    }
+
     private static List<Relation> getItemsRelation(Relation mainJobInfo, Map<String, String> idToLastestVersionMap) throws ProcessorException {
         List<Relation> itemsJobRelatedTo = new ArrayList<Relation>();
         try {
@@ -1116,7 +1275,7 @@ public class ProcessorUtilities {
 
             Set<ModuleNeeded> neededLibraries =
                     CorePlugin.getDefault().getDesignerCoreService().getNeededLibrariesForProcess(currentProcess,
-                            false);
+                            isCIMode && BitwiseOptionUtils.containOption(option, GENERATE_MAIN_ONLY));
             if (neededLibraries != null) {
                 LastGenerationInfo.getInstance().setModulesNeededWithSubjobPerJob(jobInfo.getJobId(),
                         jobInfo.getJobVersion(), neededLibraries);
@@ -1229,14 +1388,23 @@ public class ProcessorUtilities {
     public static void cleanSourceFolder(IProgressMonitor progressMonitor, IProcess currentProcess,
             IProcessor processor) {
         try {
+            ITalendProcessJavaProject jobProject = processor.getTalendJavaProject();
+            // clean up source code
             IPath codePath = processor.getSrcCodePath().removeLastSegments(2);
-            IFolder srcFolder = processor.getTalendJavaProject().getProject().getFolder(codePath);
+            IFolder srcFolder = jobProject.getProject().getFolder(codePath);
             String jobPackageFolder = JavaResourcesHelper.getJobClassPackageFolder(currentProcess);
             for (IResource resource : srcFolder.members()) {
-                if (resource.getProjectRelativePath().toPortableString().endsWith(jobPackageFolder)) {
-                    break;
+                if (!resource.getProjectRelativePath().toPortableString().endsWith(jobPackageFolder)) {
+                    resource.delete(true, progressMonitor);
+                }
+            }
+            // clean up resources folder if needed
+            if (ProcessorUtilities.isExportConfig() && !designerCoreService.isNeedContextInJar(currentProcess)) {
+                for (IResource resource : jobProject.getResourcesFolder().members()) {
+                    if (resource.exists()) {
+                        resource.delete(true, progressMonitor);
+                    }
                 }
-                resource.delete(true, progressMonitor);
             }
         } catch (CoreException e) {
             ExceptionHandler.process(e);
@@ -2605,6 +2773,39 @@ public class ProcessorUtilities {
         }
         return false;
     }
+    
+    public static boolean isEsbJob(IProcess process) {
+        return isEsbJob(process, false);
+    }
+
+    public static boolean isEsbJob(IProcess process, boolean checkCurrentProcess) {
+
+        if (process instanceof IProcess2) {
+
+            if (checkCurrentProcess) {
+                for (INode n : process.getGraphicalNodes()) {
+                    if (isEsbComponentName(n.getComponent().getName())) {
+                        return true;
+                    }
+                }
+            } else {
+                Set<JobInfo> infos = ProcessorUtilities.getChildrenJobInfo(((IProcess2) process).getProperty().getItem(), false);
+
+                for (JobInfo jobInfo : infos) {
+                    ProcessType processType = jobInfo.getProcessItem().getProcess();
+                    EList<NodeType> nodes = processType.getNode();
+                    for (NodeType nodeType : nodes) {
+                        if (isEsbComponentName(nodeType.getComponentName())) {
+                            return true;
+                        }
+                    }
+                }
+            }
+            return false;
+        }
+
+        return false;
+    }
 
     private static void addEsbJob(JobInfo jobInfo) {
         if (esbJobs.contains(esbJobKey(jobInfo.getJobId(), jobInfo.getJobVersion()))) {
@@ -2642,4 +2843,9 @@ public class ProcessorUtilities {
     public static boolean isNeedProjectProcessId(String componentName) {
         return "tRunJob".equalsIgnoreCase(componentName) || "cTalendJob".equalsIgnoreCase(componentName);
     }
+
+    public static void setCIMode(boolean isCIMode) {
+        ProcessorUtilities.isCIMode = isCIMode;
+    }
+
 }

main/plugins/org.talend.datatools.xml/src/main/java/org/talend/datatools/xml/utils/ATreeNode.java

@@ -277,6 +277,9 @@ public class ATreeNode {
      * @throws OdaException
      */
     public void setDataType(String type) throws OdaException {
+        if (type == null) {
+            type = "";
+        }
         originalDataType = new String(type);
         this.dataType = getDataType(type);
     }

main/plugins/org.talend.datatools.xml/src/main/java/org/talend/datatools/xml/utils/XSDPopulationUtil2.java

@@ -43,8 +43,10 @@ import org.eclipse.xsd.XSDImport;
 import org.eclipse.xsd.XSDModelGroup;
 import org.eclipse.xsd.XSDParticle;
 import org.eclipse.xsd.XSDSchema;
+import org.eclipse.xsd.XSDSimpleTypeDefinition;
 import org.eclipse.xsd.XSDTerm;
 import org.eclipse.xsd.XSDTypeDefinition;
+import org.eclipse.xsd.XSDVariety;
 import org.eclipse.xsd.impl.XSDNamedComponentImpl;
 import org.eclipse.xsd.util.XSDConstants;
 import org.eclipse.xsd.util.XSDResourceImpl;
@@ -355,12 +357,15 @@ public class XSDPopulationUtil2 implements IXSDPopulationUtil {
                 }
             }
             if (!resolvedAsComplex) {
-                String dataType = xsdElementDeclarationParticle.getTypeDefinition().getQName();
+                XSDTypeDefinition typeDefinition = xsdElementDeclarationParticle.getTypeDefinition();
+                String dataType = typeDefinition.getQName();
                 if (!XSDConstants
-                        .isSchemaForSchemaNamespace(xsdElementDeclarationParticle.getTypeDefinition().getTargetNamespace())
-                        && xsdElementDeclarationParticle.getTypeDefinition().getBaseType() != null) {
-                    if (!"xs:anySimpleType".equals(xsdElementDeclarationParticle.getTypeDefinition().getBaseType().getQName())) {
-                        dataType = xsdElementDeclarationParticle.getTypeDefinition().getBaseType().getQName();
+                        .isSchemaForSchemaNamespace(typeDefinition.getTargetNamespace())
+                        && typeDefinition.getBaseType() != null) {
+                    if (!"xs:anySimpleType".equals(typeDefinition.getBaseType().getQName())) {
+                        dataType = typeDefinition.getBaseType().getQName();
+                    } else if (typeDefinition instanceof XSDSimpleTypeDefinition) {
+                        dataType = getVarietyType(((XSDSimpleTypeDefinition) typeDefinition).getVariety());
                     }
                 }
                 partNode.setDataType(dataType);
@@ -378,6 +383,16 @@ public class XSDPopulationUtil2 implements IXSDPopulationUtil {
         }
     }
 
+    private String getVarietyType(XSDVariety variety) {
+        String dataType = "xs:anyType";
+        if (XSDVariety.LIST_LITERAL.equals(variety)) {
+            dataType = "xs:list";
+        } else if (XSDVariety.UNION_LITERAL.equals(variety)) {
+            dataType = "xs:union";
+        }
+        return dataType;
+    }
+
     private void handleOptionalAttribute(ATreeNode node, XSDParticle xsdParticle) {
         if (node == null || xsdParticle == null || xsdParticle.getElement() == null) {
             return;
@@ -485,6 +500,17 @@ public class XSDPopulationUtil2 implements IXSDPopulationUtil {
                 }
                 if (xsdTypeDefinition instanceof XSDComplexTypeDefinition) {
                     addComplexTypeDetails(xsdSchema, node, xsdTypeDefinition, prefix, namespace, "/" + elementName + "/");
+                } else if (xsdTypeDefinition instanceof XSDSimpleTypeDefinition) {
+                    String dataType = xsdTypeDefinition.getQName();
+                    if (!XSDConstants.isSchemaForSchemaNamespace(xsdTypeDefinition.getTargetNamespace())
+                            && xsdTypeDefinition.getBaseType() != null) {
+                        if (!"xs:anySimpleType".equals(xsdTypeDefinition.getBaseType().getQName())) {
+                            dataType = xsdTypeDefinition.getBaseType().getQName();
+                        } else if (xsdTypeDefinition instanceof XSDSimpleTypeDefinition) {
+                            dataType = getVarietyType(((XSDSimpleTypeDefinition) xsdTypeDefinition).getVariety());
+                        }
+                    }
+                    node.setDataType(dataType);
                 }
                 List<String> namespaceList = new ArrayList(namespaceToPrefix.keySet());
                 Collections.reverse(namespaceList);
@@ -565,7 +591,11 @@ public class XSDPopulationUtil2 implements IXSDPopulationUtil {
             particleToTreeNode.clear();
         }
 
-        return rootNodes.get(0);
+        if (rootNodes.isEmpty()) {
+            return null;
+        } else {
+            return rootNodes.get(0);
+        }
     }
 
     private void addSubstitutionDetails(XSDSchema xsdSchema, ATreeNode parentNode, XSDElementDeclaration elementDeclaration,
@@ -760,11 +790,14 @@ public class XSDPopulationUtil2 implements IXSDPopulationUtil {
             ATreeNode childNode = new ATreeNode();
             childNode.setValue(attributeDeclarationName);
             childNode.setType(ATreeNode.ATTRIBUTE_TYPE);
-            String dataType = xsdAttributeDeclaration.getTypeDefinition().getQName();
-            XSDTypeDefinition baseType = xsdAttributeDeclaration.getTypeDefinition().getBaseType();
-            if (!XSDConstants.isSchemaForSchemaNamespace(xsdAttributeDeclaration.getTypeDefinition().getTargetNamespace())) {
+            XSDSimpleTypeDefinition typeDefinition = xsdAttributeDeclaration.getTypeDefinition();
+            String dataType = typeDefinition.getQName();
+            XSDTypeDefinition baseType = typeDefinition.getBaseType();
+            if (!XSDConstants.isSchemaForSchemaNamespace(typeDefinition.getTargetNamespace())) {
                 if (baseType != null && !"xs:anySimpleType".equals(baseType.getQName())) { //$NON-NLS-1$
                     dataType = baseType.getQName();
+                } else if (typeDefinition instanceof XSDSimpleTypeDefinition) {
+                    dataType = getVarietyType(((XSDSimpleTypeDefinition) typeDefinition).getVariety());
                 }
             }
             if (dataType != null && dataType.length() > 0) {

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/RepositorySystemFactory.java

@@ -35,6 +35,7 @@ import org.eclipse.aether.util.artifact.SubArtifact;
 import org.eclipse.aether.util.listener.ChainedRepositoryListener;
 import org.eclipse.aether.util.listener.ChainedTransferListener;
 import org.eclipse.aether.util.repository.AuthenticationBuilder;
+import org.talend.designer.maven.aether.util.TalendAetherProxySelector;
 
 /**
  * created by wchen on Aug 10, 2017 Detailled comment
@@ -72,6 +73,8 @@ public class RepositorySystemFactory {
                     .setLocalRepositoryManager(system.newLocalRepositoryManager(repositorySystemSession, localRepo));
             repositorySystemSession.setTransferListener(new ChainedTransferListener());
             repositorySystemSession.setRepositoryListener(new ChainedRepositoryListener());
+            repositorySystemSession.setProxySelector(new TalendAetherProxySelector());
+            sessions.put(localRepo, repositorySystemSession);
         }
 
         return repositorySystemSession;
@@ -107,6 +110,7 @@ public class RepositorySystemFactory {
         Authentication auth = new AuthenticationBuilder().addUsername(userName).addPassword(password).build();
         RemoteRepository distRepo = new RemoteRepository.Builder(repositoryId, "default", repositoryUrl).setAuthentication(auth)
                 .build();
+        distRepo = new RemoteRepository.Builder(distRepo).setProxy(new TalendAetherProxySelector().getProxy(distRepo)).build();
 
         deployRequest.setRepository(distRepo);
 

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/DynamicDistributionAetherUtils.java

@@ -152,6 +152,7 @@ public class DynamicDistributionAetherUtils {
             builder = builder.setAuthentication(auth);
         }
         RemoteRepository central = builder.build();
+        central = new RemoteRepository.Builder(central).setProxy(new TalendAetherProxySelector().getProxy(central)).build();
 
         CollectRequest collectRequest = new CollectRequest();
         collectRequest.setRoot(dependency);
@@ -307,6 +308,7 @@ public class DynamicDistributionAetherUtils {
             builder = builder.setAuthentication(auth);
         }
         RemoteRepository central = builder.build();
+        central = new RemoteRepository.Builder(central).setProxy(new TalendAetherProxySelector().getProxy(central)).build();
 
         VersionRangeRequest verRangeRequest = new VersionRangeRequest();
         verRangeRequest.addRepository(central);
@@ -353,6 +355,7 @@ public class DynamicDistributionAetherUtils {
             builder = builder.setAuthentication(auth);
         }
         RemoteRepository central = builder.build();
+        central = new RemoteRepository.Builder(central).setProxy(new TalendAetherProxySelector().getProxy(central)).build();
 
         VersionRangeRequest verRangeRequest = new VersionRangeRequest();
         verRangeRequest.addRepository(central);
@@ -426,6 +429,7 @@ public class DynamicDistributionAetherUtils {
 
         LocalRepository localRepo = new LocalRepository(repositoryPath);
         session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo));
+        session.setProxySelector(new TalendAetherProxySelector());
 
         updateDependencySelector(session, monitor);
 

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/MavenLibraryResolverProvider.java

@@ -14,6 +14,7 @@ package org.talend.designer.maven.aether.util;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Consumer;
 
 import org.apache.maven.model.License;
 import org.apache.maven.model.Model;
@@ -36,6 +37,7 @@ import org.eclipse.aether.transport.file.FileTransporterFactory;
 import org.eclipse.aether.transport.http.HttpTransporterFactory;
 import org.eclipse.aether.util.repository.AuthenticationBuilder;
 import org.eclipse.m2e.core.MavenPlugin;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.nexus.ArtifactRepositoryBean;
 import org.talend.core.nexus.TalendLibsServerManager;
 import org.talend.core.runtime.maven.MavenArtifact;
@@ -82,6 +84,8 @@ public class MavenLibraryResolverProvider {
             defaultRemoteRepository = new RemoteRepository.Builder("talend", "default", talendServer.getRepositoryURL()) //$NON-NLS-1$ //$NON-NLS-2$
                     .setAuthentication(authentication).build();
         }
+        defaultRemoteRepository = new RemoteRepository.Builder(defaultRemoteRepository)
+                .setProxy(new TalendAetherProxySelector().getProxy(defaultRemoteRepository)).build();
     }
 
     public ArtifactResult resolveArtifact(MavenArtifact aritfact) throws Exception {
@@ -121,16 +125,28 @@ public class MavenLibraryResolverProvider {
     }
 
     public RemoteRepository getRemoteRepositroy(MavenArtifact aritfact) {
+        RemoteRepository remoteRepository = null;
+        Consumer<RemoteRepository> update = null;
         if (aritfact != null && aritfact.getRepositoryUrl() != null) {
             if (urlToRepositoryMap.containsKey(aritfact.getRepositoryUrl())) {
-                return urlToRepositoryMap.get(aritfact.getRepositoryUrl());
+                remoteRepository = urlToRepositoryMap.get(aritfact.getRepositoryUrl());
+            } else {
+                remoteRepository = buildRemoteRepository(aritfact);
+                urlToRepositoryMap.put(aritfact.getRepositoryUrl(), remoteRepository);
             }
-
-            RemoteRepository repository = buildRemoteRepository(aritfact);
-            urlToRepositoryMap.put(aritfact.getRepositoryUrl(), repository);
-            return repository;
+            update = (r) -> urlToRepositoryMap.put(aritfact.getRepositoryUrl(), r);
+        } else {
+            remoteRepository = defaultRemoteRepository;
+            update = (r) -> defaultRemoteRepository = r;
         }
-        return defaultRemoteRepository;
+        try {
+            remoteRepository = new RemoteRepository.Builder(remoteRepository)
+                    .setProxy(defaultRepoSystemSession.getProxySelector().getProxy(remoteRepository)).build();
+            update.accept(remoteRepository);
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+        return remoteRepository;
     }
 
     private RemoteRepository buildRemoteRepository(MavenArtifact aritfact) {
@@ -143,6 +159,8 @@ public class MavenLibraryResolverProvider {
             repository = new RemoteRepository.Builder("talend", "default", aritfact.getRepositoryUrl()) //$NON-NLS-1$ //$NON-NLS-2$
                     .setAuthentication(authentication).build();
         }
+        repository = new RemoteRepository.Builder(repository).setProxy(new TalendAetherProxySelector().getProxy(repository))
+                .build();
         return repository;
     }
 
@@ -160,6 +178,7 @@ public class MavenLibraryResolverProvider {
 
         LocalRepository localRepo = new LocalRepository( /* "target/local-repo" */target);
         session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo));
+        session.setProxySelector(new TalendAetherProxySelector());
 
         return session;
     }

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/TalendAetherProxySelector.java

@@ -0,0 +1,98 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.designer.maven.aether.util;
+
+import org.eclipse.aether.repository.Authentication;
+import org.eclipse.aether.repository.Proxy;
+import org.eclipse.aether.repository.ProxySelector;
+import org.eclipse.aether.repository.RemoteRepository;
+import org.eclipse.aether.util.repository.AuthenticationBuilder;
+import org.eclipse.aether.util.repository.DefaultProxySelector;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.TalendProxySelector;
+
+
+/**
+ * DOC cmeng  class global comment. Detailled comment
+ */
+public class TalendAetherProxySelector implements ProxySelector {
+
+    private boolean isTalendDebug = false;
+
+    public TalendAetherProxySelector() {
+        isTalendDebug = Boolean.valueOf(System.getProperty(TalendProxySelector.PROP_PRINT_LOGS, "false"));
+    }
+
+    @Override
+    public Proxy getProxy(RemoteRepository repository) {
+        /**
+         * Update each time in case the settings are changed
+         */
+        Proxy proxy = createProxySelector().getProxy(repository);
+        if (isTalendDebug) {
+            try {
+                if (repository != null) {
+                    String proxyStr = "";
+                    if (proxy != null) {
+                        proxyStr = proxy.getType() + " " + proxy.toString() + ", proxy user: "
+                                + (proxy.getAuthentication() != null ? "..." : "<empty>");
+                    }
+                    ExceptionHandler.log("Aether proxy> host: " + repository.getHost() + ", proxy: " + proxyStr);
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+        }
+        return proxy;
+    }
+
+    private DefaultProxySelector createProxySelector() {
+        DefaultProxySelector proxySelector = new DefaultProxySelector();
+        javaDefaultProxy(proxySelector);
+        return proxySelector;
+    }
+
+    private void javaDefaultProxy(DefaultProxySelector proxySelector) {
+        String[] schemas = new String[] { "http", "https" };
+        for (String schema : schemas) {
+            Proxy proxy = createProxy(schema);
+            if (proxy != null) {
+                proxySelector.add(proxy, System.getProperty(schema + ".nonProxyHosts"));
+            }
+        }
+    }
+
+    private org.eclipse.aether.repository.Proxy createProxy(String schema) {
+        String proxyHost = System.getProperty(schema + ".proxyHost");
+        if (proxyHost == null) {
+            return null;
+        }
+        String proxyUser = System.getProperty(schema + ".proxyUser");
+        String proxyPassword = System.getProperty(schema + ".proxyPassword");
+        int proxyPort = Integer.parseInt(System.getProperty(schema + ".proxyPort", "8080"));
+
+        Authentication authentication = createAuthentication(proxyUser, proxyPassword);
+        org.eclipse.aether.repository.Proxy proxyObj = new org.eclipse.aether.repository.Proxy(schema, proxyHost, proxyPort,
+                authentication);
+        return proxyObj;
+    }
+
+    private Authentication createAuthentication(String proxyUser, String proxyPassword) {
+        Authentication authentication = null;
+        if (proxyUser != null) {
+            authentication = new AuthenticationBuilder().addUsername(proxyUser).addPassword(proxyPassword).build();
+        }
+        return authentication;
+    }
+
+}

main/plugins/org.talend.designer.maven.tos/resources/tacokit/pom.xml

@@ -10,7 +10,7 @@
 	<artifactId>studio-tacokit-dependencies</artifactId>
 	<packaging>pom</packaging>
 	<properties>
-		<tacokit.components.version>1.1.0</tacokit.components.version>
+		<tacokit.components.version>1.2.3</tacokit.components.version>
 	</properties>
 	<repositories>
 		<repository>

main/plugins/org.talend.designer.maven.tos/resources/tcompv1/pom.xml

@@ -11,7 +11,7 @@
     <packaging>pom</packaging>
     
     <properties>
-        <tcomp.version>1.1.10</tcomp.version>
+        <tcomp.version>1.1.14</tcomp.version>
         <slf4j.version>1.7.25</slf4j.version>
     </properties>
     

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/MavenUIService.java

@@ -14,6 +14,7 @@ package org.talend.designer.maven.ui;
 
 import java.util.Dictionary;
 import java.util.Hashtable;
+import java.util.Set;
 
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IProgressMonitor;
@@ -118,6 +119,17 @@ public class MavenUIService implements IMavenUIService {
         String studioUserSettingsFile = MavenPlugin.getMavenConfiguration().getUserSettingsFile();
         // apply the user settings to MavenResolver
         Dictionary<String, String> props = new Hashtable<String, String>();
+        Set<Object> keySet = System.getProperties().keySet();
+        if (keySet != null) {
+            for (Object keyObj : keySet) {
+                if (keyObj instanceof String) {
+                    String key = keyObj.toString();
+                    if (key.startsWith("org.ops4j.pax.url.mvn.")) {
+                        props.put(key, System.getProperty(key));
+                    }
+                }
+            }
+        }
         if (studioUserSettingsFile != null && !"".equals(studioUserSettingsFile)) {
             // change back to use the user settings after Pax-url-eather fix the space bug
             props.put("org.ops4j.pax.url.mvn.settings", studioUserSettingsFile);

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/setting/preference/M2eUserSettingForTalendLoginTask.java

@@ -294,20 +294,25 @@ public class M2eUserSettingForTalendLoginTask extends AbstractLoginTask {
         boolean isLocal = isLocalRepository();
         IPath localRepoPath = null;
         if (!isLocal) {
-            String mvnHome = System.getenv("M2_HOME"); //$NON-NLS-1$
-            if (mvnHome == null) {
-                mvnHome = System.getenv("MAVEN_HOME"); //$NON-NLS-1$
-            }
-            if (StringUtils.isNotBlank(mvnHome)) {
-                File globalSettings = new File(mvnHome).toPath().resolve("conf").resolve("settings.xml").toFile(); //$NON-NLS-1$ //$NON-NLS-2$
-                if (globalSettings.exists()) {
-                    DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
-                    Document document = builder.parse(globalSettings);
-                    Node node = document.getElementsByTagName("localRepository").item(0); //$NON-NLS-1$
-                    if (node != null) {
-                        String repoPath = node.getTextContent();
-                        if (StringUtils.isNotBlank(repoPath)) {
-                            localRepoPath = new Path(repoPath);
+            String customMavenRepoistory = System.getProperty("maven.local.repository");
+            if (customMavenRepoistory != null) {
+                localRepoPath = new Path(customMavenRepoistory);
+            } else {
+                String mvnHome = System.getenv("M2_HOME"); //$NON-NLS-1$
+                if (mvnHome == null) {
+                    mvnHome = System.getenv("MAVEN_HOME"); //$NON-NLS-1$
+                }
+                if (StringUtils.isNotBlank(mvnHome)) {
+                    File globalSettings = new File(mvnHome).toPath().resolve("conf").resolve("settings.xml").toFile(); //$NON-NLS-1$ //$NON-NLS-2$
+                    if (globalSettings.exists()) {
+                        DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
+                        Document document = builder.parse(globalSettings);
+                        Node node = document.getElementsByTagName("localRepository").item(0); //$NON-NLS-1$
+                        if (node != null) {
+                            String repoPath = node.getTextContent();
+                            if (StringUtils.isNotBlank(repoPath)) {
+                                localRepoPath = new Path(repoPath);
+                            }
                         }
                     }
                 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/launch/MavenCommandLauncher.java

@@ -13,6 +13,8 @@
 package org.talend.designer.maven.launch;
 
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.commons.lang.StringUtils;
 import org.eclipse.core.resources.IContainer;
@@ -95,6 +97,12 @@ public abstract class MavenCommandLauncher {
 
     private Map<String, Object> argumentsMap;
 
+    private static final String REGEX_TEST_CASE_FAILURES_STR = "\\[ERROR\\] Tests run:.*\\[ERROR\\] There are test\\p{Print}+\n";
+
+    private static final Pattern REGEX_TEST_CASE_FAILURES = Pattern.compile(REGEX_TEST_CASE_FAILURES_STR, Pattern.DOTALL);
+
+    private boolean ignoreTestFailure = false;
+
     public MavenCommandLauncher(String goals) {
         super();
         Assert.isNotNull(goals);
@@ -103,6 +111,14 @@ public abstract class MavenCommandLauncher {
         this.debugOutput = MavenPlugin.getMavenConfiguration().isDebugOutput();
     }
 
+    public boolean isIgnoreTestFailure() {
+        return ignoreTestFailure;
+    }
+
+    public void setIgnoreTestFailure(boolean ignoreTestFailure) {
+        this.ignoreTestFailure = ignoreTestFailure;
+    }
+
     protected String getGoals() {
         return goals;
     }
@@ -205,6 +221,12 @@ public abstract class MavenCommandLauncher {
                 workingCopy.setAttribute(IJavaLaunchConfigurationConstants.ATTR_VM_ARGUMENTS, vmargs);
             }
 
+            // ignore test failures
+            if (this.ignoreTestFailure) {
+                workingCopy.setAttribute(IJavaLaunchConfigurationConstants.ATTR_VM_ARGUMENTS, "-Dmaven.test.failure.ignore=true "
+                        + workingCopy.getAttribute(IJavaLaunchConfigurationConstants.ATTR_VM_ARGUMENTS, ""));
+            }
+
             String programArgs = getArgumentValue(TalendProcessArgumentConstant.ARG_PROGRAM_ARGUMENTS);
             if (StringUtils.isNotEmpty(programArgs)) {
                 workingCopy.setAttribute(IJavaLaunchConfigurationConstants.ATTR_PROGRAM_ARGUMENTS, programArgs);
@@ -318,7 +340,18 @@ public abstract class MavenCommandLauncher {
 
         if (TalendMavenConstants.GOAL_INSTALL.equals(launchConfiguration.getAttribute(MavenLaunchConstants.ATTR_GOALS, ""))) {
             if (errors.length() != 0) {
-                throw new Exception(errors.toString());
+                String remainingErr = errors.toString();
+                if (this.ignoreTestFailure) {
+                    Matcher m = REGEX_TEST_CASE_FAILURES.matcher(errors);
+                    int matchIdx = 0;
+                    while (m.find()) {
+                        matchIdx = m.end();
+                    }
+                    remainingErr = errors.substring(matchIdx);
+                }
+                if (remainingErr.trim().length() > 0) {
+                    throw new Exception(remainingErr.toString());
+                }
             }
         }
     }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/AggregatorPomsHelper.java

@@ -77,6 +77,7 @@ import org.talend.core.runtime.services.IFilterService;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.designer.core.ICamelDesignerCoreService;
 import org.talend.designer.maven.launch.MavenPomCommandLauncher;
+import org.talend.designer.maven.model.MavenSystemFolders;
 import org.talend.designer.maven.model.TalendJavaProjectConstants;
 import org.talend.designer.maven.model.TalendMavenConstants;
 import org.talend.designer.maven.template.MavenTemplateManager;
@@ -494,6 +495,10 @@ public class AggregatorPomsHelper {
         return null;
     }
 
+    public IFolder getCodeSrcFolder(ERepositoryObjectType codeType) {
+        return getCodeFolder(codeType).getFolder(MavenSystemFolders.JAVA.getPath());
+    }
+
     public IFolder getProcessFolder(ERepositoryObjectType type) {
         return getProjectPomsFolder().getFolder(DIR_JOBS).getFolder(type.getFolder());
     }
@@ -684,6 +689,23 @@ public class AggregatorPomsHelper {
         new ProgressMonitorDialog(Display.getDefault().getActiveShell()).run(true, true, runnableWithProgress);
     }
 
+    public void syncParentJobPomsForPropertyChange(Property property) {
+        IRunProcessService runProcessService = getRunProcessService();
+        ProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
+        List<Relation> itemsHaveRelationWith = RelationshipItemBuilder.getInstance().getItemsHaveRelationWith(property.getId(),
+                property.getVersion());
+        try {
+            for (Relation relation : itemsHaveRelationWith) {
+                IRepositoryViewObject object = factory.getSpecificVersion(relation.getId(), relation.getVersion(), true);
+                if (runProcessService != null) {
+                    runProcessService.generatePom(object.getProperty().getItem());
+                }
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
     private String getModulePath(IFile pomFile) {
         IFile parentPom = getProjectRootPom();
         if (parentPom != null) {

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/BuildCacheManager.java

@@ -223,8 +223,13 @@ public class BuildCacheManager {
 
                 String goal = (String) argumentsMap.get(TalendProcessArgumentConstant.ARG_GOAL);
                 MavenPomCommandLauncher mavenLauncher = new MavenPomCommandLauncher(pomFile, goal);
-                mavenLauncher.setSkipTests(true);
                 mavenLauncher.setArgumentsMap(argumentsMap);
+                if (isBuildJob()) {
+                    mavenLauncher.setIgnoreTestFailure(true);
+                } else {
+                    // run job, still skip tests anyway
+                    mavenLauncher.setSkipTests(true);
+                }
                 mavenLauncher.execute(monitor);
             } finally {
                 deleteBuildAggregatorPom();
@@ -375,4 +380,13 @@ public class BuildCacheManager {
         return aggregatorPomsHelper;
     }
 
+    private boolean isBuildJob() {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
+            IRunProcessService service = (IRunProcessService) GlobalServiceRegister.getDefault()
+                    .getService(IRunProcessService.class);
+            return service.isExportConfig();
+        }
+        return false;
+    }
+
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/AbstractMavenProcessorPom.java

@@ -220,6 +220,29 @@ public abstract class AbstractMavenProcessorPom extends CreateMavenBundleTemplat
                     MavenArtifact mvnArtifact = MavenUrlHelper.parseMvnUrl(module.getMavenUri());
                     include.setValue(mvnArtifact.getGroupId() + ":" + mvnArtifact.getArtifactId()); //$NON-NLS-1$
                 }
+                
+                //removing digital signatures from uber jar
+                Xpp3Dom filters = new Xpp3Dom("filters"); //$NON-NLS-1$
+                Xpp3Dom filter = new Xpp3Dom("filter"); //$NON-NLS-1$
+                Xpp3Dom artifact = new Xpp3Dom("artifact"); //$NON-NLS-1$
+                artifact.setValue("*:*");
+                Xpp3Dom filterExcludes = new Xpp3Dom("excludes"); //$NON-NLS-1$
+                Xpp3Dom excludeSF = new Xpp3Dom("exclude");
+                excludeSF.setValue("META-INF/*.SF");
+                Xpp3Dom excludeDSA = new Xpp3Dom("exclude");
+                excludeDSA.setValue("META-INF/*.DSA");
+                Xpp3Dom excludeRSA = new Xpp3Dom("exclude");
+                excludeRSA.setValue("META-INF/*.RSA");
+
+                filterExcludes.addChild(excludeSF);
+                filterExcludes.addChild(excludeDSA);
+                filterExcludes.addChild(excludeRSA);
+                
+                filter.addChild(artifact);
+                filter.addChild(filterExcludes);
+                filters.addChild(filter);
+                configuration.addChild(filters);
+                
                 plugins.add(shade);
             }
         }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenJobPom.java

@@ -874,29 +874,32 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
             IMaven maven = MavenPlugin.getMaven();
             ArtifactRepository repository = maven.getLocalRepository();
             Node filesNode = document.getElementsByTagName("files").item(0);
-            for (Entry<String, Set<Dependency>> entry : duplicateDependencies.entrySet()) {
-                Set<Dependency> dependencies = entry.getValue();
-                for (Dependency dependency : dependencies) {
-                    String sourceLocation = maven
-                            .getArtifactPath(repository, dependency.getGroupId(), dependency.getArtifactId(),
-                                    dependency.getVersion(), dependency.getType(), dependency.getClassifier());
-                    Path path = new File(repository.getBasedir()).toPath().resolve(sourceLocation);
-                    sourceLocation = path.toString();
-                    String destName = path.getFileName().toString();
-                    Node fileNode = document.createElement("file");
-                    filesNode.appendChild(fileNode);
-
-                    Node sourcesNode = document.createElement("source");
-                    sourcesNode.setTextContent(sourceLocation);
-                    fileNode.appendChild(sourcesNode);
-
-                    Node outputDirNode = document.createElement("outputDirectory");
-                    outputDirNode.setTextContent("lib");
-                    fileNode.appendChild(outputDirNode);
-
-                    Node destNameNode = document.createElement("destName");
-                    destNameNode.setTextContent(destName);
-                    fileNode.appendChild(destNameNode);
+            // TESB-27614:NPE while building a route
+            if (filesNode != null) {
+                for (Entry<String, Set<Dependency>> entry : duplicateDependencies.entrySet()) {
+                    Set<Dependency> dependencies = entry.getValue();
+                    for (Dependency dependency : dependencies) {
+                        String sourceLocation = maven
+                                .getArtifactPath(repository, dependency.getGroupId(), dependency.getArtifactId(),
+                                        dependency.getVersion(), dependency.getType(), dependency.getClassifier());
+                        Path path = new File(repository.getBasedir()).toPath().resolve(sourceLocation);
+                        sourceLocation = path.toString();
+                        String destName = path.getFileName().toString();
+                        Node fileNode = document.createElement("file");
+                        filesNode.appendChild(fileNode);
+    
+                        Node sourcesNode = document.createElement("source");
+                        sourcesNode.setTextContent(sourceLocation);
+                        fileNode.appendChild(sourcesNode);
+    
+                        Node outputDirNode = document.createElement("outputDirectory");
+                        outputDirNode.setTextContent("lib");
+                        fileNode.appendChild(outputDirNode);
+    
+                        Node destNameNode = document.createElement("destName");
+                        destNameNode.setTextContent(destName);
+                        fileNode.appendChild(destNameNode);
+                    }
                 }
             }
         } catch (CoreException e) {

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenJobletPom.java

@@ -123,7 +123,9 @@ public class CreateMavenJobletPom extends AbstractMavenProcessorPom {
     @Override
     protected void addDependencies(Model model) {
         try {
-            getProcessorDependenciesManager().updateDependencies(null, model);
+            if (ERepositoryObjectType.JOBLET == ERepositoryObjectType.getType(getJobProcessor().getProperty())) {
+                getProcessorDependenciesManager().updateDependencies(null, model);
+            }
             addChildrenDependencies(model.getDependencies());
         } catch (ProcessorException e) {
             ExceptionHandler.process(e);

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenStandardJobOSGiPom.java

@@ -152,11 +152,13 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
         model.addProperty("talend.job.finalName", "${talend.job.name}-bundle-${project.version}");
         Build build = model.getBuild();
         
-        IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IESBService.class)) {
+            IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
 
-        if (isServiceOperation || service.isRESTService((ProcessItem) getJobProcessor().getProperty().getItem())
-                || isRouteOperation(getJobProcessor().getProperty())) {
-            build.addPlugin(addSkipDockerMavenPlugin());
+            if (isServiceOperation || service.isRESTService((ProcessItem) getJobProcessor().getProperty().getItem())
+                    || isRouteOperation(getJobProcessor().getProperty())) {
+                build.addPlugin(addSkipDockerMavenPlugin());
+            }
         }
         
         if (isServiceOperation) {
@@ -229,9 +231,19 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
             Set<JobInfo> childrenJobInfo = getJobProcessor().getBuildChildrenJobs();
             for (JobInfo jobInfo : childrenJobInfo) {
                 Property property = jobInfo.getProcessItem().getProperty();
-                String coordinate = getCoordinate(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo),
+
+                String bundle = "";
+
+                if (!jobInfo.isJoblet()) {
+                    property = jobInfo.getProcessItem().getProperty();
+                    if ("OSGI".equals(property.getAdditionalProperties().get(TalendProcessArgumentConstant.ARG_BUILD_TYPE))) {
+                        bundle = "-bundle";
+                    }
+                }
+
+                String coordinate = getCoordinate(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo) + bundle,
                         MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(property));
-                Dependency dependency = getDependencyObject(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo), PomIdsHelper.getJobVersion(property),
+                Dependency dependency = getDependencyObject(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo) + bundle, PomIdsHelper.getJobVersion(property),
                                 MavenConstants.PACKAGING_JAR, null);
                 jobCoordinateMap.put(coordinate, dependency);
             }
@@ -242,7 +254,7 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
                 PomIdsHelper.getJobArtifactId(parentProperty), MavenConstants.PACKAGING_JAR,
                 PomIdsHelper.getJobVersion(parentProperty));
         Dependency parentDependency = getDependencyObject(PomIdsHelper.getJobGroupId(parentProperty), PomIdsHelper.getJobArtifactId(parentProperty), PomIdsHelper.getJobVersion(parentProperty),
-                        MavenConstants.PACKAGING_JAR, null);
+                MavenConstants.PACKAGING_JAR, null);
         jobCoordinateMap.put(parentCoordinate, parentDependency);
         try {
             Document document = PomUtil.loadAssemblyFile(null, assemblyFile);
@@ -283,25 +295,25 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
         List<IRepositoryViewObject> serviceRepoList = null;
 
         boolean isDataServiceOperation = false;
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IESBService.class)) {
+            IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
+            try {
+                IProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
+                serviceRepoList = factory.getAll(ERepositoryObjectType.valueOf(ERepositoryObjectType.class, "SERVICES"));
 
-        IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
-
-        try {
-            IProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
-            serviceRepoList = factory.getAll(ERepositoryObjectType.valueOf(ERepositoryObjectType.class, "SERVICES"));
-
-            for (IRepositoryViewObject serviceItem : serviceRepoList) {
-                if (service != null) {
-                    List<String> jobIds = service.getSerivceRelatedJobIds(serviceItem.getProperty().getItem());
-                    if (jobIds.contains(property.getId())) {
-                        isDataServiceOperation = true;
-                        break;
+                for (IRepositoryViewObject serviceItem : serviceRepoList) {
+                    if (service != null) {
+                        List<String> jobIds = service.getSerivceRelatedJobIds(serviceItem.getProperty().getItem());
+                        if (jobIds.contains(property.getId())) {
+                            isDataServiceOperation = true;
+                            break;
+                        }
                     }
                 }
-            }
 
-        } catch (PersistenceException e) {
-            ExceptionHandler.process(e);
+            } catch (PersistenceException e) {
+                ExceptionHandler.process(e);
+            }
         }
 
         return isDataServiceOperation;
@@ -311,32 +323,33 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
         List<IRepositoryViewObject> routeRepoList = null;
 
         boolean isRouteOperation = false;
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IESBService.class)) {
 
-        IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
+            IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
 
-        try {
-            IProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
-            routeRepoList = factory.getAll(ERepositoryObjectType.valueOf(ERepositoryObjectType.class, "ROUTE"));
+            try {
+                IProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
+                routeRepoList = factory.getAll(ERepositoryObjectType.valueOf(ERepositoryObjectType.class, "ROUTE"));
 
-            for (IRepositoryViewObject routeItem : routeRepoList) {
-                if (service != null) {
+                for (IRepositoryViewObject routeItem : routeRepoList) {
+                    if (service != null) {
 
-                    List<Relation> relations = RelationshipItemBuilder.getInstance().getItemsRelatedTo(routeItem.getId(),
-                            routeItem.getVersion(), RelationshipItemBuilder.JOB_RELATION);
-                    for (Relation relation : relations) {
-                        if (relation.getType() == RelationshipItemBuilder.JOB_RELATION) {
-                            if (relation.getId().equals(property.getId())) {
-                                isRouteOperation = true;
+                        List<Relation> relations = RelationshipItemBuilder.getInstance().getItemsRelatedTo(routeItem.getId(),
+                                routeItem.getVersion(), RelationshipItemBuilder.JOB_RELATION);
+                        for (Relation relation : relations) {
+                            if (relation.getType() == RelationshipItemBuilder.JOB_RELATION) {
+                                if (relation.getId().equals(property.getId())) {
+                                    isRouteOperation = true;
+                                }
                             }
                         }
                     }
                 }
+
+            } catch (PersistenceException e) {
+                ExceptionHandler.process(e);
             }
-
-        } catch (PersistenceException e) {
-            ExceptionHandler.process(e);
         }
-
         return isRouteOperation;
     }
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/PomUtil.java

@@ -511,6 +511,10 @@ public class PomUtil {
         return repoPath + "/" + artifactPath; //$NON-NLS-1$
     }
 
+    public static String getLocalRepositoryPath() {
+        return MavenPlugin.getMaven().getLocalRepositoryPath();
+    }
+
     /**
      * Get absolute path for installed artifact
      *
@@ -1127,4 +1131,20 @@ public class PomUtil {
                     || pathname.getName().equals("m2e-lastUpdated.properties"); //$NON-NLS-1$
         }
     };
+
+    public static boolean checkIfJobDependencyExist(IFile parentJobPom, Property property) throws Exception {
+        boolean found = false;
+        Model model = MavenPlugin.getMavenModelManager().readMavenModel(parentJobPom);
+        String groupId = PomIdsHelper.getJobGroupId(property);
+        String artifactId = PomIdsHelper.getJobArtifactId(property);
+        String version = PomIdsHelper.getJobVersion(property);
+        for (Dependency dependency : model.getDependencies()) {
+            if (dependency.getGroupId().equals(groupId) && dependency.getArtifactId().equals(artifactId)
+                    && dependency.getVersion().equals(version)) {
+                found = true;
+                break;
+            }
+        }
+        return found;
+    }
 }

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/utils/RemoteModulesHelper.java

@@ -34,6 +34,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
@@ -207,6 +208,7 @@ public class RemoteModulesHelper {
                     for (String groupId : snapshotgroupIds) {
                         List<MavenArtifact> searchResults = customerRepHandler.search(groupId, null, null, false, true);
                         monitor.worked(10);
+                        replaceTimeStampWithSnapShot(searchResults);
                         addModulesToCache(mavenUristoSearch, searchResults, localCache);
                     }
                 }
@@ -216,6 +218,24 @@ public class RemoteModulesHelper {
             }
         }
 
+        private void replaceTimeStampWithSnapShot(List<MavenArtifact> searchResults) {
+            if (searchResults != null && searchResults.size() > 0) {
+                for (MavenArtifact m : searchResults) {
+                    String version = m.getVersion();
+                    if (version != null && !version.contains(MavenConstants.SNAPSHOT)
+                            && timeStampRegex(version)) {
+                        version = version.substring(0, version.indexOf("-")) + MavenConstants.SNAPSHOT;
+                        m.setVersion(version);
+                    }
+                }
+            }
+        }
+
+        private boolean timeStampRegex(String version) {
+            boolean match = Pattern.matches("\\S*[-]{1}[0-9]{8}[.]{1}[0-9]{6}[-]{1}\\S*", version);
+            return match;
+        }
+
         private void searchFromLocalDataFile(Set<String> mavenUristoSearch, IProgressMonitor monitor) {
             LibraryDataService service = LibraryDataService.getInstance();
             List<MavenArtifact> artifactList = new ArrayList<MavenArtifact>();

main/plugins/org.talend.librariesmanager/src/main/java/messages.properties

@@ -26,5 +26,6 @@ ModulesNeededProvider.Module.Exception=Needed module list do not contains this m
 
 ShareLibsJob.message=Share libraries from maven to {0} server
 ShareLibsJob.sharingLibraries=Sharing libraries : {0}
+ShareLibsJob.monitor.cancelled=User cancelled.
 
 ExtensionModuleManager.moduleGroup.cycleReference=Module group {0} was cycle referenced!

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/maven/ShareLibrareisHelper.java

@@ -13,26 +13,35 @@
 package org.talend.librariesmanager.maven;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.codehaus.jackson.map.util.ISO8601Utils;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Status;
 import org.eclipse.core.runtime.SubMonitor;
 import org.eclipse.core.runtime.jobs.Job;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.model.general.ModuleNeeded;
-import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.RepositoryArtifactHandlerManager;
 import org.talend.core.nexus.TalendLibsServerManager;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.librariesmanager.i18n.Messages;
+import org.talend.librariesmanager.nexus.utils.VersionUtil;
 
 /**
  * created by Talend on 2015年7月31日 Detailled comment
@@ -49,9 +58,7 @@ public abstract class ShareLibrareisHelper {
         IStatus status = Status.OK_STATUS;
         // deploy to maven if needed and share to custom nexus
         try {
-            int searchLimit = 50;
             setJobName(job, Messages.getString("ShareLibsJob.message", TYPE_NEXUS));
-            final List<MavenArtifact> searchResults = new ArrayList<MavenArtifact>();
             ArtifactRepositoryBean customNexusServer = TalendLibsServerManager.getInstance().getCustomNexusServer();
             IRepositoryArtifactHandler customerRepHandler = RepositoryArtifactHandlerManager
                     .getRepositoryHandler(customNexusServer);
@@ -60,69 +67,104 @@ public abstract class ShareLibrareisHelper {
                 if (filesToShare == null) {
                     return Status.CANCEL_STATUS;
                 }
-                SubMonitor mainSubMonitor = SubMonitor.convert(monitor, filesToShare.size());
 
                 // collect groupId to search
                 Set<String> groupIds = new HashSet<String>();
+                Map<String, List<MavenArtifact>> snapshotArtifactMap = new HashMap<String, List<MavenArtifact>>();
+                Map<String, List<MavenArtifact>> releaseArtifactMap = new HashMap<String, List<MavenArtifact>>();
+                Set<String> snapshotGroupIdSet = new HashSet<String>();
+                Set<String> releaseGroupIdSet = new HashSet<String>();
                 for (ModuleNeeded module : filesToShare.keySet()) {
+                    checkCancel(monitor);
                     if (module.getMavenUri() != null) {
                         MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(module.getMavenUri());
                         if (parseMvnUrl != null) {
                             groupIds.add(parseMvnUrl.getGroupId());
+                            if (isSnapshotVersion(parseMvnUrl.getVersion())) {
+                                snapshotGroupIdSet.add(parseMvnUrl.getGroupId());
+                            } else {
+                                releaseGroupIdSet.add(parseMvnUrl.getGroupId());
+                            }
                         }
                     }
                 }
+                List<MavenArtifact> searchResults = new ArrayList<MavenArtifact>();
                 for (String groupId : groupIds) {
-                    searchResults.addAll(customerRepHandler.search(groupId, null, null, true, true));
+                    checkCancel(monitor);
+                    if (releaseGroupIdSet.contains(groupId)) {
+                        searchResults = customerRepHandler.search(groupId, null, null, true, false);
+                        if (searchResults != null) {
+                            for (MavenArtifact result : searchResults) {
+                                checkCancel(monitor);
+                                putArtifactToMap(result, releaseArtifactMap, false);
+                            }
+                        }
+                    }
+                    checkCancel(monitor);
+                    if (snapshotGroupIdSet.contains(groupId)) {
+                        searchResults = customerRepHandler.search(groupId, null, null, false, true);
+                        if (searchResults != null) {
+                            for (MavenArtifact result : searchResults) {
+                                checkCancel(monitor);
+                                putArtifactToMap(result, snapshotArtifactMap, true);
+                            }
+                        }
+                    }
                 }
-
-                int limit = searchLimit;
-                int shareIndex = 0;
                 Iterator<ModuleNeeded> iterator = filesToShare.keySet().iterator();
+                Map<File, MavenArtifact> shareFiles = new HashMap<>();
                 while (iterator.hasNext()) {
-                    if (monitor.isCanceled()) {
-                        return Status.CANCEL_STATUS;
-                    }
-                    shareIndex++;
-                    if (shareIndex == limit) {
-                        limit += searchLimit;
-                    }
-
+                    checkCancel(monitor);
                     ModuleNeeded next = iterator.next();
                     File file = filesToShare.get(next);
-                    String name = file.getName();
                     MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(next.getMavenUri());
                     if (artifact == null) {
                         continue;
                     }
-
-                    boolean eixst = false;
-                    String groupId = artifact.getGroupId();
-                    String artifactId = artifact.getArtifactId();
-                    String version = artifact.getVersion();
-                    for (MavenArtifact remoteAtifact : searchResults) {
-                        String rGroup = remoteAtifact.getGroupId();
-                        String rArtifact = remoteAtifact.getArtifactId();
-                        String rVersion = remoteAtifact.getVersion();
-                        if (groupId != null && artifactId != null && version != null && groupId.equals(rGroup)
-                                && artifactId.equals(rArtifact) && version.equals(rVersion)) {
-                            eixst = true;
-                            break;
+                    try {
+                        Integer.parseInt(artifact.getType());
+                        // FIXME unexpected type if it's an integer, should fix it in component module definition.
+                        continue;
+                    } catch (NumberFormatException e) {
+                        //
+                    }
+                    boolean isSnapshotVersion = isSnapshotVersion(artifact.getVersion());
+                    String key = getArtifactKey(artifact, isSnapshotVersion);
+                    List<MavenArtifact> artifactList = null;
+                    if (isSnapshotVersion) {
+                        artifactList = snapshotArtifactMap.get(key);
+                    } else {
+                        artifactList = releaseArtifactMap.get(key);
+                        // skip checksum for release artifact.
+                        if (artifactList != null && artifactList.contains(artifact)
+                                && !Boolean.getBoolean("force_libs_release_update")) {
+                            continue;
                         }
                     }
-                    if (eixst) {
-                        continue;
+                    if (artifactList != null && artifactList.size() > 0) {
+                        if (isSameFileWithRemote(file, artifactList, customNexusServer)) {
+                            continue;
+                        }
                     }
-                    mainSubMonitor.setTaskName(Messages.getString("ShareLibsJob.sharingLibraries", name));
-
+                    shareFiles.put(file, artifact);
+                }
+                SubMonitor mainSubMonitor = SubMonitor.convert(monitor, shareFiles.size());
+                for (Map.Entry<File, MavenArtifact> entry : shareFiles.entrySet()) {
+                    checkCancel(monitor);
                     try {
-                        shareToRepository(file, artifact);
+                        File k = entry.getKey();
+                        MavenArtifact v = entry.getValue();
+                        mainSubMonitor.setTaskName(Messages.getString("ShareLibsJob.sharingLibraries", k.getName()));
+                        shareToRepository(k, v);
                         mainSubMonitor.worked(1);
                     } catch (Exception e) {
-                        continue;
+                        ExceptionHandler.process(e);
                     }
                 }
             }
+        } catch (InterruptedException e) {
+            ExceptionHandler.process(e);
+            status = Status.CANCEL_STATUS;
         } catch (Exception e) {
             status = new Status(IStatus.ERROR, "unknown", IStatus.ERROR, "Share libraries failed !", e);
         }
@@ -131,12 +173,104 @@ public abstract class ShareLibrareisHelper {
 
     }
 
+    public void putArtifactToMap(MavenArtifact artifact, Map<String, List<MavenArtifact>> map, boolean isShapshot) {
+        String key = getArtifactKey(artifact, isShapshot);
+        List<MavenArtifact> list = map.get(key);
+        if (list == null) {
+            list = new ArrayList<MavenArtifact>();
+            map.put(key, list);
+        }
+        list.add(artifact);
+    }
+
+    private String getArtifactKey(MavenArtifact artifact, boolean isShapshot) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(artifact.getGroupId()).append("-");
+        sb.append(artifact.getArtifactId()).append("-");
+        String version = artifact.getVersion();
+        if (isShapshot) {
+            version = VersionUtil.getSNAPSHOTVersion(version);
+        }
+        sb.append(version);
+        if (StringUtils.isNotEmpty(artifact.getClassifier())) {
+            sb.append("-").append(artifact.getClassifier());
+        }
+        return sb.toString();
+    }
+
+    private boolean isSameFileWithRemote(File localFile, List<MavenArtifact> artifactList,
+            ArtifactRepositoryBean customNexusServer) throws Exception {
+        String localFileShaCode = DigestUtils.shaHex(new FileInputStream(localFile));
+        MavenArtifact lastUpdatedArtifact = null;
+        if (ArtifactRepositoryBean.NexusType.ARTIFACTORY.name().equalsIgnoreCase(customNexusServer.getType())) {
+            lastUpdatedArtifact = getLateUpdatedMavenArtifact(artifactList);
+        } else {
+            lastUpdatedArtifact = artifactList.stream().max(Comparator.comparing(e -> e.getVersion())).get();
+        }
+        if (lastUpdatedArtifact != null && StringUtils.equals(localFileShaCode, lastUpdatedArtifact.getSha1())) {
+            return true;
+        }
+        return false;
+    }
+
+    private MavenArtifact getLateUpdatedMavenArtifact(List<MavenArtifact> artifactList) {
+        if (artifactList.size() == 1) {
+            return artifactList.get(0);
+        }
+        MavenArtifact latestVersion = null;
+        Date lastUpdate = null;
+        for (MavenArtifact art : artifactList) {
+            if (latestVersion == null) {
+                if (art.getLastUpdated() != null) {
+                    latestVersion = art;
+                    lastUpdate = parsetDate(art.getLastUpdated());
+                }
+            } else if (art.getLastUpdated() != null && lastUpdate != null) {
+                Date artLastUpdate = parsetDate(art.getLastUpdated());
+                if (artLastUpdate != null && lastUpdate.getTime() < artLastUpdate.getTime()) {
+                    latestVersion = art;
+                    lastUpdate = artLastUpdate;
+                }
+            }
+        }
+        if (latestVersion != null) {
+            return latestVersion;
+        } else {
+            return artifactList.get(artifactList.size() - 1);
+        }
+    }
+
+    private Date parsetDate(String strDate) {
+        Date date = null;
+        if (strDate != null) {
+            try {
+                date = ISO8601Utils.parse(strDate);
+            } catch (Exception ex) {
+                ExceptionHandler.process(ex);
+            }
+        }
+        return date;
+    }
+
+    private boolean isSnapshotVersion(String version) {
+        if (version != null && version.toUpperCase().endsWith(MavenUrlHelper.VERSION_SNAPSHOT)) {
+            return true;
+        }
+        return false;
+    }
+
     private void setJobName(Job job, String jobName) {
         if (job != null) {
             job.setName(jobName);
         }
     }
 
+    protected void checkCancel(IProgressMonitor monitor) throws InterruptedException {
+        if (monitor.isCanceled()) {
+            throw new InterruptedException(Messages.getString("ShareLibsJob.monitor.cancelled"));
+        }
+    }
+
     public abstract Map<ModuleNeeded, File> getFilesToShare(IProgressMonitor monitor);
 
     public abstract void shareToRepository(File jarFile, MavenArtifact module) throws Exception;

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/ModulesNeededProvider.java

@@ -706,8 +706,13 @@ public class ModulesNeededProvider {
                 ModuleNeeded toAdd = new ModuleNeeded(context, currentImport.getMODULE(), currentImport.getMESSAGE(),
                         isRequired);
                 toAdd.setMavenUri(currentImport.getMVN());
-                if (!isRequired && "BeanItem".equals(routine.eClass().getName())) {
-                	toAdd.getExtraAttributes().put("IS_OSGI_EXCLUDED", Boolean.TRUE);
+                if (!isRequired) {
+                    if ("BeanItem".equals(routine.eClass().getName())) {
+                        toAdd.getExtraAttributes().put("IS_OSGI_EXCLUDED", Boolean.TRUE);
+                    }
+                    if ("RoutineItem".equals(routine.eClass().getName())) {
+                        continue;
+                    }
                 }
                 // toAdd.setStatus(ELibraryInstallStatus.INSTALLED);
                 importNeedsList.add(toAdd);

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/LocalLibraryManager.java

@@ -224,7 +224,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         }
     }
 
-    private void guessMavenRUIFromIndex(File jarFile, Map<String, String> sourceAndMavenUri) {
+    public void guessMavenRUIFromIndex(File jarFile, Map<String, String> sourceAndMavenUri) {
         // TODO????? should deploy with all versions
         String urisFromIndex = LibrariesIndexManager.getInstance().getMavenLibIndex().getJarsToRelativePath()
                 .get(jarFile.getName());
@@ -1594,7 +1594,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         return isSameFile(jarFile, jarInLib);
     }
 
-    private boolean isSameFile(File f1, File f2) {
+    public boolean isSameFile(File f1, File f2) {
         if (f1 == null || f2 == null) {
             return false;
         }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/AbstractArtifactRepositoryHandler.java

@@ -13,9 +13,13 @@
 package org.talend.librariesmanager.nexus;
 
 import java.io.File;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.util.Dictionary;
 import java.util.Hashtable;
 
+import org.apache.commons.lang.StringUtils;
 import org.talend.core.nexus.ArtifactRepositoryBean;
 import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.NexusConstants;
@@ -62,10 +66,22 @@ public abstract class AbstractArtifactRepositoryHandler implements IRepositoryAr
         if (props == null) {
             props = new Hashtable<String, String>();
         }
-        String repositories = null;
-        String custom_server = serverBean.getServer();
+
         String custom_user = serverBean.getUserName();
         String custom_pass = serverBean.getPassword();
+        try {
+            if (StringUtils.isNotBlank(custom_user)) {
+                custom_user = URLEncoder.encode(custom_user, StandardCharsets.UTF_8.toString());
+            }
+            if (StringUtils.isNotBlank(custom_pass)) {
+                custom_pass = URLEncoder.encode(custom_pass, StandardCharsets.UTF_8.toString());
+            }
+        } catch (UnsupportedEncodingException e1) {
+            throw new RuntimeException(e1);
+        }
+
+        String repositories = null;
+        String custom_server = serverBean.getServer();
         String release_rep = serverBean.getRepositoryId();
         String snapshot_rep = serverBean.getSnapshotRepId();
         if (custom_server.endsWith(NexusConstants.SLASH)) {

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/ArtifacoryRepositoryHandler.java

@@ -14,6 +14,7 @@ package org.talend.librariesmanager.nexus;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -25,11 +26,15 @@ import org.apache.http.client.fluent.Request;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.params.CoreConnectionPNames;
 import org.apache.http.util.EntityUtils;
 import org.eclipse.m2e.core.MavenPlugin;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.TalendProxySelector.IProxySelectorProvider;
+import org.talend.core.nexus.HttpClientTransport;
 import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.NexusConstants;
+import org.talend.core.nexus.NexusServerUtils;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.designer.maven.aether.RepositorySystemFactory;
@@ -43,9 +48,9 @@ import net.sf.json.JSONObject;
  */
 public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandler {
 
-    private String SEARCH_SERVICE = "api/search/gavc?";
+    private String SEARCH_SERVICE = "api/search/gavc?"; //$NON-NLS-1$
 
-    private String SEARCH_RESULT_PREFIX = "api/storage/";
+    private String SEARCH_RESULT_PREFIX = "api/storage/";//$NON-NLS-1$
 
     /*
      * (non-Javadoc)
@@ -80,17 +85,30 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
     }
 
     private boolean doConnectionCheck(String repositoryUrl) throws ClientProtocolException, IOException {
-        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword();
-        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes()));
-        Header authority = new BasicHeader("Authorization", basicAuth);
+        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword(); //$NON-NLS-1$
+        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes())); //$NON-NLS-1$
+        Header authority = new BasicHeader("Authorization", basicAuth); //$NON-NLS-1$
         HttpGet get = new HttpGet(repositoryUrl);
         get.addHeader(authority);
         DefaultHttpClient httpclient = new DefaultHttpClient();
-        HttpResponse response = httpclient.execute(get);
-        if (response.getStatusLine().getStatusCode() == 200) {
-            return true;
+        httpclient.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, NexusServerUtils.getTimeout());
+        httpclient.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, NexusServerUtils.getTimeout());
+        IProxySelectorProvider proxySelector = null;
+        try {
+            try {
+                proxySelector = HttpClientTransport.addProxy(httpclient, new URI(repositoryUrl));
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+            HttpResponse response = httpclient.execute(get);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                return true;
+            }
+            return false;
+        } finally {
+            HttpClientTransport.removeProxy(proxySelector);
+            httpclient.getConnectionManager().shutdown();
         }
-        return false;
     }
 
     /*
@@ -103,51 +121,53 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
     public List<MavenArtifact> search(String groupIdToSearch, String artifactId, String versionToSearch, boolean fromRelease,
             boolean fromSnapshot) throws Exception {
         String serverUrl = serverBean.getServer();
-        if (!serverUrl.endsWith("/")) {
-            serverUrl = serverUrl + "/";
+        if (!serverUrl.endsWith("/")) { //$NON-NLS-1$
+            serverUrl = serverUrl + "/"; //$NON-NLS-1$
         }
         String searchUrl = serverUrl + SEARCH_SERVICE;
 
-        String repositoryId = "";
+        String repositoryId = ""; //$NON-NLS-1$
         if (fromRelease) {
             repositoryId = serverBean.getRepositoryId();
         }
         if (fromSnapshot) {
-            if ("".equals(repositoryId)) {
+            if ("".equals(repositoryId)) { //$NON-NLS-1$
                 repositoryId = serverBean.getSnapshotRepId();
             } else {
-                repositoryId = repositoryId + "," + serverBean.getSnapshotRepId();
+                repositoryId = repositoryId + "," + serverBean.getSnapshotRepId(); //$NON-NLS-1$
             }
         }
         String query = "";//$NON-NLS-1$
-        if (!"".equals(repositoryId)) {
+        if (!"".equals(repositoryId)) { //$NON-NLS-1$
             query = "repos=" + repositoryId;//$NON-NLS-1$
         }
         if (groupIdToSearch != null) {
-            if (!"".equals(query)) {
-                query = query + "&";
+            if (!"".equals(query)) { //$NON-NLS-1$
+                query = query + "&"; //$NON-NLS-1$
             }
             query = query + "g=" + groupIdToSearch;//$NON-NLS-1$
         }
         if (artifactId != null) {
-            if (!"".equals(query)) {
-                query = query + "&";
+            if (!"".equals(query)) { //$NON-NLS-1$
+                query = query + "&"; //$NON-NLS-1$
             }
             query = query + "a=" + artifactId;//$NON-NLS-1$
         }
 
         if (versionToSearch != null) {
-            if (!"".equals(query)) {
-                query = query + "&";
+            if (!"".equals(query)) { //$NON-NLS-1$
+                query = query + "&"; //$NON-NLS-1$
             }
             query = query + "v=" + versionToSearch;//$NON-NLS-1$
         }
         searchUrl = searchUrl + query;
         Request request = Request.Get(searchUrl);
-        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword();
-        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes()));
-        Header authority = new BasicHeader("Authorization", basicAuth);
+        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword(); //$NON-NLS-1$
+        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes())); //$NON-NLS-1$
+        Header authority = new BasicHeader("Authorization", basicAuth); //$NON-NLS-1$
         request.addHeader(authority);
+        Header resultDetailHeader = new BasicHeader("X-Result-Detail", "info"); //$NON-NLS-1$ //$NON-NLS-2$
+        request.addHeader(resultDetailHeader);
         List<MavenArtifact> resultList = new ArrayList<MavenArtifact>();
 
         HttpResponse response = request.execute().returnResponse();
@@ -155,11 +175,11 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
         if (content.isEmpty()) {
             return resultList;
         }
-        JSONObject responseObject = new JSONObject().fromObject(content);
-        String resultStr = responseObject.getString("results");
+        JSONObject responseObject = JSONObject.fromObject(content);
+        String resultStr = responseObject.getString("results"); //$NON-NLS-1$
         JSONArray resultArray = null;
         try {
-            resultArray = new JSONArray().fromObject(resultStr);
+            resultArray = JSONArray.fromObject(resultStr);
         } catch (Exception e) {
             throw new Exception(resultStr);
         }
@@ -167,46 +187,57 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
             String resultUrl = serverUrl + SEARCH_RESULT_PREFIX;
             for (int i = 0; i < resultArray.size(); i++) {
                 JSONObject jsonObject = resultArray.getJSONObject(i);
-                String uri = jsonObject.getString("uri");
+                String lastUpdated = jsonObject.getString("lastUpdated"); //$NON-NLS-1$
+                String uri = jsonObject.getString("uri"); //$NON-NLS-1$
                 uri = uri.substring(resultUrl.length(), uri.length());
-                String[] split = uri.split("/");
+                String[] split = uri.split("/"); //$NON-NLS-1$
                 if (split.length > 4) {
                     String fileName = split[split.length - 1];
-                    if (!fileName.endsWith("pom")) {
+                    if (!fileName.endsWith("pom")) { //$NON-NLS-1$
                         String type = null;
                         int dotIndex = fileName.lastIndexOf('.');
                         if (dotIndex > 0) {
                             type = fileName.substring(dotIndex + 1);
-
                         }
                         if (type != null) {
                             MavenArtifact artifact = new MavenArtifact();
                             String v = split[split.length - 2];
                             String a = split[split.length - 3];
-                            String g = "";
+                            String g = ""; //$NON-NLS-1$
                             for (int j = 1; j < split.length - 3; j++) {
-                                if ("".equals(g)) {
+                                if ("".equals(g)) { //$NON-NLS-1$
                                     g = split[j];
                                 } else {
-                                    g = g + "." + split[j];
+                                    g = g + "." + split[j]; //$NON-NLS-1$
                                 }
                             }
                             artifact.setGroupId(g);
                             artifact.setArtifactId(a);
                             artifact.setVersion(v);
                             artifact.setType(type);
+                            artifact.setLastUpdated(lastUpdated);
+                            fillChecksumData(jsonObject, artifact);
                             resultList.add(artifact);
                         }
                     }
-
                 }
-
             }
         }
-
         return resultList;
     }
 
+    private void fillChecksumData(JSONObject responseObject, MavenArtifact artifact) {
+        if (responseObject.containsKey("checksums")) { //$NON-NLS-1$
+            JSONObject checkSum = responseObject.getJSONObject("checksums"); //$NON-NLS-1$
+            if (checkSum != null && checkSum.containsKey("sha1")) { //$NON-NLS-1$
+                artifact.setSha1(checkSum.getString("sha1")); //$NON-NLS-1$
+            }
+            if (checkSum != null && checkSum.containsKey("md5")) { //$NON-NLS-1$
+                artifact.setMd5(checkSum.getString("md5")); //$NON-NLS-1$
+            }
+        }
+    }
+
     /*
      * (non-Javadoc)
      *
@@ -216,7 +247,7 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
     @Override
     public void deploy(File content, String groupId, String artifactId, String classifier, String extension, String version)
             throws Exception {
-        String repositoryId = "";
+        String repositoryId = ""; //$NON-NLS-1$
         boolean isRelease = !version.endsWith(MavenUrlHelper.VERSION_SNAPSHOT);
         if (isRelease) {
             repositoryId = serverBean.getRepositoryId();
@@ -238,7 +269,7 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
     @Override
     public void deployWithPOM(File content, File pomFile, String groupId, String artifactId, String classifier, String extension,
             String version) throws Exception {
-        String repositoryId = "";
+        String repositoryId = ""; //$NON-NLS-1$
         boolean isRelease = !version.endsWith(MavenUrlHelper.VERSION_SNAPSHOT);
         if (isRelease) {
             repositoryId = serverBean.getRepositoryId();

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/Nexus3RepositoryHandler.java

@@ -14,8 +14,11 @@ package org.talend.librariesmanager.nexus;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.http.Header;
@@ -24,10 +27,15 @@ import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.params.CoreConnectionPNames;
 import org.apache.log4j.Logger;
 import org.eclipse.m2e.core.MavenPlugin;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.TalendProxySelector.IProxySelectorProvider;
+import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.nexus.HttpClientTransport;
 import org.talend.core.nexus.IRepositoryArtifactHandler;
+import org.talend.core.nexus.NexusServerUtils;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.designer.maven.aether.RepositorySystemFactory;
@@ -48,6 +56,8 @@ public class Nexus3RepositoryHandler extends AbstractArtifactRepositoryHandler {
 
     private INexus3SearchHandler currentQueryHandler = null;
 
+    private static final Map<ArtifactRepositoryBean, INexus3SearchHandler> LAST_HANDLER_MAP = new HashMap<ArtifactRepositoryBean, INexus3SearchHandler>();
+    
     private static List<INexus3SearchHandler> queryHandlerList = new ArrayList<INexus3SearchHandler>();
 
     @Override
@@ -94,11 +104,25 @@ public class Nexus3RepositoryHandler extends AbstractArtifactRepositoryHandler {
         HttpGet get = new HttpGet(repositoryUrl);
         get.addHeader(authority);
         DefaultHttpClient httpclient = new DefaultHttpClient();
-        HttpResponse response = httpclient.execute(get);
-        if (response.getStatusLine().getStatusCode() == 200) {
-            return true;
+        
+        httpclient.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, NexusServerUtils.getTimeout());
+        httpclient.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, NexusServerUtils.getTimeout());
+        IProxySelectorProvider proxySelector = null;
+        try {
+            try {
+                proxySelector = HttpClientTransport.addProxy(httpclient, new URI(repositoryUrl));
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+            HttpResponse response = httpclient.execute(get);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                return true;
+            }
+            return false;
+        } finally {
+            HttpClientTransport.removeProxy(proxySelector);
+            httpclient.getConnectionManager().shutdown();
         }
-        return false;
     }
 
     /*
@@ -124,8 +148,12 @@ public class Nexus3RepositoryHandler extends AbstractArtifactRepositoryHandler {
 
     private List<MavenArtifact> doSearch(String repositoryId, String groupIdToSearch, String artifactId, String versionToSearch)
             throws Exception {
+        initQueryHandler();
         if (currentQueryHandler == null) {
-            currentQueryHandler = getQueryHandler();
+            currentQueryHandler = LAST_HANDLER_MAP.get(serverBean);
+            if (currentQueryHandler == null) {
+                currentQueryHandler = queryHandlerList.get(0);
+            }
         }
         List<MavenArtifact> result = new ArrayList<MavenArtifact>();
         try {
@@ -146,16 +174,16 @@ public class Nexus3RepositoryHandler extends AbstractArtifactRepositoryHandler {
                 }
             }
         }
+        LAST_HANDLER_MAP.put(serverBean, currentQueryHandler);
         return result;
     }
 
-    private INexus3SearchHandler getQueryHandler() {
+    private void initQueryHandler() {
         if (queryHandlerList.size() == 0) {
             queryHandlerList.add(new Nexus3V1SearchHandler(serverBean));
             queryHandlerList.add(new Nexus3BetaSearchHandler(serverBean));
             queryHandlerList.add(new Nexus3ScriptSearchHandler(serverBean));
         }
-        return queryHandlerList.get(0);
     }
 
     /*

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/nexus3/handler/AbsNexus3SearchHandler.java

@@ -14,11 +14,13 @@ package org.talend.librariesmanager.nexus.nexus3.handler;
 // ============================================================================
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpHeaders;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.methods.HttpGet;
@@ -60,6 +62,7 @@ public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
 
     protected abstract String getSearchUrl();
 
+    @Override
     public List<MavenArtifact> search(String repositoryId, String groupIdToSearch, String artifactId, String versionToSearch)
             throws Exception {
         List<MavenArtifact> resultList = new ArrayList<MavenArtifact>();
@@ -84,6 +87,13 @@ public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
             protected HttpResponse execute(IProgressMonitor monitor, DefaultHttpClient httpClient, URI targetURI)
                     throws Exception {
                 HttpGet httpGet = new HttpGet(targetURI);
+                String userName = serverBean.getUserName();
+                if (StringUtils.isNotBlank(userName) && !Boolean.getBoolean("talend.studio.search.nexus3.disableBasicAuth")) {
+                    String auth = userName + ":" + serverBean.getPassword();
+                    String authHeader = "Basic " + new String(Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)));
+                    httpGet.setHeader(HttpHeaders.AUTHORIZATION, authHeader);
+                    httpClient.setCredentialsProvider(null);
+                }
                 HttpResponse response = httpClient.execute(httpGet);
                 if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
                     sb.append(EntityUtils.toString(response.getEntity()));
@@ -120,8 +130,9 @@ public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
                 artifact.setGroupId(jsonObject.getString("group")); //$NON-NLS-1$
                 artifact.setArtifactId(jsonObject.getString("name")); //$NON-NLS-1$
                 artifact.setVersion(jsonObject.getString("version")); //$NON-NLS-1$
-                JSONArray assertsArray = jsonObject.getJSONArray("assets"); //$NON-NLS-1$
+                JSONArray assertsArray = jsonObject.getJSONArray("assets"); //$NON-NLS-1$                
                 artifact.setType(getPackageType(assertsArray));
+                fillCheckSumData(assertsArray, artifact);
                 resultList.add(artifact);
             }
         }
@@ -135,6 +146,9 @@ public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
             for (int i = 0; i < assertsArray.size(); i++) {
                 JSONObject jsonObject = assertsArray.getJSONObject(i);
                 String path = jsonObject.getString("path"); //$NON-NLS-1$
+                if (path != null && path.endsWith(".exe")) { //$NON-NLS-1$
+                    return "exe"; //$NON-NLS-1$
+                }
                 if (path != null && path.endsWith(".zip")) { //$NON-NLS-1$
                     return "zip"; //$NON-NLS-1$
                 }
@@ -149,6 +163,27 @@ public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
         return type;
     }
 
+    private void fillCheckSumData(JSONArray assertsArray, MavenArtifact artifact) {
+        if (assertsArray != null) {
+            for (int i = 0; i < assertsArray.size(); i++) {
+                JSONObject jsonObject = assertsArray.getJSONObject(i);
+                if (jsonObject.containsKey("path")) { //$NON-NLS-1$
+                    String path = jsonObject.getString("path"); //$NON-NLS-1$
+                    if (path != null && path.endsWith(artifact.getType())) { // $NON-NLS-1$
+                        if (jsonObject.containsKey("checksum")) {
+                            JSONObject checksumObject = jsonObject.getJSONObject("checksum"); //$NON-NLS-1$
+                            if (checksumObject != null && checksumObject.containsKey("sha1")) {//$NON-NLS-1$
+                                artifact.setSha1(checksumObject.getString("sha1")); //$NON-NLS-1$
+                                artifact.setMd5(checksumObject.getString("md5")); //$NON-NLS-1$
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     protected String getQueryParameter(String repositoryId, String groupIdToSearch, String artifactId, String versionToSearch,
             String continuationToken) {
         StringBuffer sb = new StringBuffer();

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/utils/VersionUtil.java

@@ -0,0 +1,32 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.librariesmanager.nexus.utils;
+
+import org.talend.core.runtime.maven.MavenUrlHelper;
+
+/**
+ * DOC hwang  class global comment. Detailled comment
+ */
+public class VersionUtil {
+    
+    public static String getSNAPSHOTVersion(String rVersion) {
+        if(rVersion == null) {
+            return rVersion;
+        }
+        if(rVersion.contains("-")) {
+            return rVersion.substring(0, rVersion.indexOf("-") + 1) + MavenUrlHelper.VERSION_SNAPSHOT;
+        }
+        return rVersion;
+    }
+
+}

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/utils/ConnectionContextHelper.java

@@ -764,7 +764,25 @@ public final class ConnectionContextHelper {
                     Connection hadoopClusterConnection = hadoopClusterItem.getConnection();
                     ContextItem hadoopClusterContextItem = ContextUtils.getContextItemById2(hadoopClusterConnection
                             .getContextId());
-                    Set<String> hcNeededVars = retrieveContextVar(elementParameters, hadoopClusterConnection, category, true);
+                    Set<String> hcNeededVars = retrieveContextVar(elementParameters, hadoopClusterConnection, category, false);
+                    // The tSparkConfiguration & tHadoopConfManager are singleton in process.
+                    List<INode> nodesOfType = new ArrayList<INode>();
+                    List<? extends INode> spakconfNodes = process.getNodesOfType("tSparkConfiguration"); //$NON-NLS-1$
+                    if (spakconfNodes != null && !spakconfNodes.isEmpty()) {
+                        nodesOfType.addAll(spakconfNodes);
+                    }
+                    List<? extends INode> hadoopconfNodes = process.getNodesOfType("tHadoopConfManager"); //$NON-NLS-1$
+                    if (hadoopconfNodes != null && !hadoopconfNodes.isEmpty()) {
+                        nodesOfType.addAll(hadoopconfNodes);
+                    }
+                    for (INode node : nodesOfType) {
+                        Set<String> envirNeededVars = retrieveContextVar(node.getElementParameters(), hadoopClusterConnection,
+                                category, false);
+                        if (envirNeededVars != null && !envirNeededVars.isEmpty()) {
+                            hcNeededVars.addAll(envirNeededVars);
+                        }
+                    }
+
                     List<ContextItem> contextItems = new ArrayList<>();
                     if (contextItem != null || hadoopClusterContextItem != null) {
                         // find added variables

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/utils/ExtendedNodeConnectionContextUtils.java

@@ -125,11 +125,16 @@ public class ExtendedNodeConnectionContextUtils {
         QuboleS3BucketKey,
         QuboleS3Region,
 
+        // Override hadoop configuration
+        setHadoopConf,
+        hadoopConfSpecificJar,
+
         // DataBricks
         DataBricksEndpoint,
         DataBricksClusterId,
         DataBricksToken,
         DataBricksDBFSDepFolder
+
     }
 
     static List<IContextParameter> getContextVariables(final String prefixName, Connection conn, Set<IConnParamName> paramSet) {

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/metadata/managment/hive/HiveClassLoaderFactory.java

@@ -14,15 +14,19 @@ package org.talend.metadata.managment.hive;
 
 import java.io.File;
 import java.util.Set;
+import java.util.function.Consumer;
 
+import org.apache.commons.lang3.StringUtils;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.Platform;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.ILibraryManagerService;
 import org.talend.core.classloader.ClassLoaderFactory;
 import org.talend.core.classloader.DynamicClassLoader;
 import org.talend.core.database.conn.ConnParameterKeys;
 import org.talend.core.hadoop.EHadoopConfigurationJars;
+import org.talend.core.hadoop.HadoopConfJarBean;
 import org.talend.core.hadoop.IHadoopClusterService;
 import org.talend.core.hadoop.repository.HadoopRepositoryUtil;
 import org.talend.core.model.metadata.IMetadataConnection;
@@ -121,13 +125,29 @@ public class HiveClassLoaderFactory {
         }
 
         Object[] configurationJars = new String[0];
+        Consumer<DynamicClassLoader> afterLoad = null;
         String useCustomConfs = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_USE_CUSTOM_CONFS);
         if (Boolean.valueOf(useCustomConfs)) {
             String clusterId = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_HADOOP_CLUSTER_ID);
-            String customConfsJarName = getCustomConfsJarName(clusterId);
-            if (customConfsJarName != null) {
-                configurationJars = new String[] { customConfsJarName };
+
+            HadoopConfJarBean confJarBean = getCustomConfsJarName(clusterId);
+            if (confJarBean != null) {
+                if (confJarBean.isOverrideCustomConf()) {
+                    String overrideCustomConfPath = confJarBean.getOriginalOverrideCustomConfPath();
+                    if (StringUtils.isBlank(overrideCustomConfPath) || !new File(overrideCustomConfPath).exists()) {
+                        ExceptionHandler.process(
+                                new Exception("Set Hadoop configuration JAR path is invalid: " + overrideCustomConfPath));
+                    } else {
+                        afterLoad = (t) -> t.addLibrary(overrideCustomConfPath);
+                    }
+                } else {
+                    String customConfsJarName = confJarBean.getCustomConfJarName();
+                    if (customConfsJarName != null) {
+                        configurationJars = new String[] { customConfsJarName };
+                    }
+                }
             }
+
         } else {
             String useKrb = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_USE_KRB);
             if (Boolean.valueOf(useKrb)) {
@@ -158,12 +178,15 @@ public class HiveClassLoaderFactory {
                 loader.addLibrary(jarFile.getAbsolutePath());
             }
         }
+        if (afterLoad != null) {
+            afterLoad.accept(loader);
+        }
     }
 
-    private static String getCustomConfsJarName(String clusterId) {
+    private static HadoopConfJarBean getCustomConfsJarName(String clusterId) {
         IHadoopClusterService hadoopClusterService = HadoopRepositoryUtil.getHadoopClusterService();
         if (hadoopClusterService != null) {
-            return hadoopClusterService.getCustomConfsJarName(clusterId);
+            return hadoopClusterService.getCustomConfsJar(clusterId).orElse(null);
         }
         return null;
     }

main/plugins/org.talend.repository.items.importexport/plugin.xml

@@ -15,5 +15,9 @@
             id="org.talend.repository.items.sync.children.contexts"
             provider="org.talend.repository.items.importexport.handlers.exports.SyncChildrenContextsResourcesProvider">
       </resourcesProvider>
+      <resourcesProvider
+            id="org.talend.repository.items.importexport.handlers.exports.SyncChildrenTestReportsProvider"
+            provider="org.talend.repository.items.importexport.handlers.exports.SyncChildrenTestReportsProvider">
+      </resourcesProvider>
    </extension>
 </plugin>

main/plugins/org.talend.repository.items.importexport/src/main/java/org/talend/repository/items/importexport/handlers/exports/SyncChildrenTestReportsProvider.java

@@ -0,0 +1,78 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.repository.items.importexport.handlers.exports;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
+import org.eclipse.core.resources.IFolder;
+import org.eclipse.core.resources.IResource;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.talend.commons.runtime.utils.io.FileCopyUtils;
+import org.talend.core.GlobalServiceRegister;
+import org.talend.core.model.properties.ProcessItem;
+import org.talend.core.runtime.process.ITalendProcessJavaProject;
+import org.talend.core.runtime.repository.build.IBuildResourcesProvider;
+import org.talend.core.runtime.util.ParametersUtil;
+import org.talend.designer.runprocess.IRunProcessService;
+
+public class SyncChildrenTestReportsProvider implements IBuildResourcesProvider {
+
+    private static final String TEST_REPORTS_FOLDER = "surefire-reports";
+
+    @Override
+    public void prepare(IProgressMonitor monitor, Map<String, Object> parameters) throws Exception {
+        if (parameters == null) {
+            return;
+        }
+        final ITalendProcessJavaProject processJavaProject = (ITalendProcessJavaProject) ParametersUtil.getObject(parameters,
+                OBJ_PROCESS_JAVA_PROJECT, ITalendProcessJavaProject.class);
+        if (processJavaProject == null) {
+            return;
+        }
+        final ProcessItem processItem = (ProcessItem) ParametersUtil.getObject(parameters, OBJ_PROCESS_ITEM, ProcessItem.class);
+        if (processItem == null) {
+            return;
+        }
+        final List dependenciesItems = (List) ParametersUtil.getObject(parameters, OBJ_ITEM_DEPENDENCIES, List.class);
+        if (dependenciesItems == null || dependenciesItems.isEmpty()) {
+            return;
+        }
+        if (!GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
+            return;
+        }
+        final IRunProcessService runProcessService = (IRunProcessService) GlobalServiceRegister.getDefault()
+                .getService(IRunProcessService.class);
+
+        final File reportTargetFolder = processJavaProject.getTargetFolder().getFolder(TEST_REPORTS_FOLDER).getLocation()
+                .toFile();
+
+        for (Object item : dependenciesItems) {
+            if (item instanceof ProcessItem) {
+                ITalendProcessJavaProject childJavaProject = runProcessService
+                        .getTalendJobJavaProject(((ProcessItem) item).getProperty());
+                if (childJavaProject != null) {
+                    childJavaProject.getTargetFolder().refreshLocal(IResource.DEPTH_INFINITE, monitor);
+                    final IFolder childReportFolder = childJavaProject.getTargetFolder().getFolder(TEST_REPORTS_FOLDER);
+                    if (childReportFolder.exists()) {
+                        FileCopyUtils.syncFolder(childReportFolder.getLocation().toFile(), reportTargetFolder, false);
+                    }
+                }
+            }
+        }
+
+        processJavaProject.getTargetFolder().refreshLocal(IResource.DEPTH_INFINITE, monitor);
+    }
+
+}

main/plugins/org.talend.repository.metadata/src/main/java/org/talend/repository/ui/wizards/metadata/connection/files/xml/TreePopulator.java

@@ -235,7 +235,7 @@ public class TreePopulator extends AbstractTreePopulator {
                 ExceptionHandler.process(e);
                 return false;
             }
-            if (treeNode == null || treeNode.getChildren().length == 0) {
+            if (treeNode == null) {
                 return false;
             } else {
                 availableXmlTree.setInput(new Object[] { treeNode });

main/plugins/org.talend.updates.runtime/src/main/java/org/talend/updates/runtime/engine/component/ComponentP2ExtraFeature.java

@@ -32,11 +32,13 @@ import org.eclipse.equinox.p2.core.IProvisioningAgent;
 import org.eclipse.equinox.p2.core.IProvisioningAgentProvider;
 import org.eclipse.equinox.p2.core.ProvisionException;
 import org.eclipse.equinox.p2.engine.IPhaseSet;
+import org.eclipse.equinox.p2.engine.IProfile;
 import org.eclipse.equinox.p2.engine.PhaseSetFactory;
 import org.eclipse.equinox.p2.metadata.IInstallableUnit;
 import org.eclipse.equinox.p2.operations.InstallOperation;
 import org.eclipse.equinox.p2.operations.ProfileModificationJob;
 import org.eclipse.equinox.p2.operations.ProvisioningSession;
+import org.eclipse.equinox.p2.operations.UninstallOperation;
 import org.eclipse.equinox.p2.query.IQuery;
 import org.eclipse.equinox.p2.query.QueryUtil;
 import org.osgi.framework.Bundle;
@@ -137,7 +139,8 @@ public class ComponentP2ExtraFeature extends P2ExtraFeature implements IP2Compon
 
         IProvisioningAgent agent = null;
         try {
-            agent = agentProvider.createAgent(getP2AgentUri());
+            IProfile profile = getProfile(agentProvider, progress);
+            agent = profile.getProvisioningAgent();
 
             updateRoamingProp(agent, agentProvider);
 
@@ -153,27 +156,50 @@ public class ComponentP2ExtraFeature extends P2ExtraFeature implements IP2Compon
                         getVersion(), Arrays.toString(allRepoUris.toArray(new URI[allRepoUris.size()])));
             }
 
+            IPhaseSet talendPhaseSet = PhaseSetFactory
+                    .createDefaultPhaseSetExcluding(new String[] { PhaseSetFactory.PHASE_CHECK_TRUST });
+            // uninstall first
+            Collection<IInstallableUnit> installedP2Feature = getInstalledP2Feature(profile, toInstall, subMonitor);
+            ProvisioningSession session = new ProvisioningSession(agent);
+            if (installedP2Feature != null && !installedP2Feature.isEmpty()) {
+                UninstallOperation uninstallOperation = new UninstallOperation(session, installedP2Feature);
+                uninstallOperation.setProfileId(getP2ProfileId());
+                IStatus resolveModal = uninstallOperation.resolveModal(subMonitor);
+                if (resolveModal.getSeverity() == IStatus.ERROR) {
+                    return Messages.createErrorStatus(null, "ComponentP2ExtraFeature.error.installing.new.components", //$NON-NLS-1$
+                            uninstallOperation.getResolutionDetails());
+                }
+                ProfileModificationJob provisioningJob = (ProfileModificationJob) uninstallOperation
+                        .getProvisioningJob(subMonitor.newChild(1));
+                if (subMonitor.isCanceled()) {
+                    return Messages.createCancelStatus("ComponentP2ExtraFeature.user.cancel.installation.of.components", //$NON-NLS-1$
+                            getP2IuId(), getVersion());
+                }
+                if (provisioningJob == null) {
+                    return Messages.createErrorStatus(null, "ComponentP2ExtraFeature.error.installing.new.components", //$NON-NLS-1$
+                            getP2IuId(), getVersion(), uninstallOperation.getResolutionDetails());
+                }
+                provisioningJob.setPhaseSet(talendPhaseSet);
+                provisioningJob.run(subMonitor.newChild(1));
+                agent.stop();
+                // update profile
+                profile = getProfile(agentProvider, subMonitor);
+                agent = profile.getProvisioningAgent();
+                updateRoamingProp(agent, agentProvider);
+                session = new ProvisioningSession(agent);
+            }
+
             // install
-            InstallOperation installOperation = new InstallOperation(new ProvisioningSession(agent), toInstall);
+            InstallOperation installOperation = new InstallOperation(session, toInstall);
             installOperation.setProfileId(getP2ProfileId());
             IStatus installResolvedStatus = installOperation.resolveModal(subMonitor.newChild(1));
-            if (subMonitor.isCanceled()) {
-                return Messages.createCancelStatus("ComponentP2ExtraFeature.user.cancel.installation.of.components", //$NON-NLS-1$
-                        getP2IuId(), getVersion());
-            }
             if (installResolvedStatus.getSeverity() == IStatus.ERROR) {
                 return Messages.createErrorStatus(null, "ComponentP2ExtraFeature.error.installing.new.components", getP2IuId(), //$NON-NLS-1$
                         getVersion(), installOperation.getResolutionDetails());
             } // else perform the installlation
-            IPhaseSet talendPhaseSet = PhaseSetFactory
-                    .createDefaultPhaseSetExcluding(new String[] { PhaseSetFactory.PHASE_CHECK_TRUST });
 
             ProfileModificationJob provisioningJob = (ProfileModificationJob) installOperation.getProvisioningJob(subMonitor
                     .newChild(1));
-            if (subMonitor.isCanceled()) {
-                return Messages.createCancelStatus("ComponentP2ExtraFeature.user.cancel.installation.of.components", //$NON-NLS-1$
-                        getP2IuId(), getVersion());
-            }
             if (provisioningJob == null) {
                 return Messages.createErrorStatus(null, "ComponentP2ExtraFeature.error.installing.new.components", //$NON-NLS-1$
                         getP2IuId(), getVersion(), installOperation.getResolutionDetails());
@@ -196,6 +222,9 @@ public class ComponentP2ExtraFeature extends P2ExtraFeature implements IP2Compon
             return Messages.createErrorStatus(e,
                     "ComponentP2ExtraFeature.error.installing.components.uri.exception", getP2IuId(), //$NON-NLS-1$
                     getVersion());
+        } catch (Exception e) {
+            return Messages.createErrorStatus(e, "ComponentP2ExtraFeature.error.installing.components.uri.exception", getP2IuId(), //$NON-NLS-1$
+                    getVersion());
         } finally {
             if (agent != null) {// agent creation did not fail
                 removeAllRepositories(agent, allRepoUris);

main/plugins/org.talend.updates.runtime/src/main/java/org/talend/updates/runtime/model/AbstractExtraFeature.java

@@ -90,6 +90,7 @@ public abstract class AbstractExtraFeature implements ExtraFeature {
         this.degradable = degradable;
         this.types = types;
         this.categories = categories;
+        this.share = getRepositoryService().getProxyRepositoryFactory().isFullLogonFinished();
     }
 
     @Override

main/plugins/org.talend.updates.runtime/src/main/java/org/talend/updates/runtime/model/ExtraFeature.java

@@ -26,6 +26,8 @@ import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.OperationCanceledException;
 import org.eclipse.swt.graphics.Image;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.core.GlobalServiceRegister;
+import org.talend.repository.model.IRepositoryService;
 import org.talend.updates.runtime.feature.model.Category;
 import org.talend.updates.runtime.feature.model.Type;
 import org.talend.updates.runtime.model.InstallationStatus.Status;
@@ -242,4 +244,7 @@ public interface ExtraFeature extends Comparable<Object> {
         return 0;
     }
 
+    default IRepositoryService getRepositoryService() {
+        return (IRepositoryService) GlobalServiceRegister.getDefault().getService(IRepositoryService.class);
+    }
 }

main/plugins/org.talend.updates.runtime/src/main/java/org/talend/updates/runtime/model/P2ExtraFeature.java

@@ -24,6 +24,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -161,28 +162,8 @@ public class P2ExtraFeature extends AbstractExtraFeature implements IP2Feature {
         try {
 
             IQuery<IInstallableUnit> iuQuery = QueryUtil.createIUQuery(p2IuId2);
-            boolean interrupted = false;
-            IProfile profile = null;
-            // there seems to be a bug because if the agent is created too quickly then the profile is empty.
-            // so we loop until we get a proper profile
-            final String p2ProfileId = getP2ProfileId();
-            do {
-                try {
-                    Thread.sleep(50);
-                } catch (InterruptedException e) {
-                    interrupted = true;
-                }
-                if (agent != null) {
-                    agent.stop();
-                }
-                agent = agentProvider.createAgent(getP2AgentUri());
-                IProfileRegistry profRegistry = (IProfileRegistry) agent.getService(IProfileRegistry.SERVICE_NAME);
-                profile = profRegistry.getProfile(p2ProfileId);
-            } while (profile != null && profile.getTimestamp() == 0 && !interrupted && !subMonitor.isCanceled());
-
-            if (profile == null || subMonitor.isCanceled()) {
-                throw new ProvisionException("Could not find the p2 profile named " + p2ProfileId); //$NON-NLS-1$
-            }
+            IProfile profile = getProfile(agentProvider, subMonitor);
+            agent = profile.getProvisioningAgent();
             subMonitor.worked(1);
             IQueryResult<IInstallableUnit> iuQueryResult = profile.available(iuQuery, subMonitor.newChild(1));
             if (subMonitor.isCanceled()) {
@@ -196,6 +177,33 @@ public class P2ExtraFeature extends AbstractExtraFeature implements IP2Feature {
         }
     }
 
+    protected IProfile getProfile(IProvisioningAgentProvider agentProvider, IProgressMonitor progress) throws ProvisionException {
+        IProvisioningAgent agent = null;
+        IProfile profile = null;
+        boolean interrupted = false;
+        // there seems to be a bug because if the agent is created too quickly then the profile is empty.
+        // so we loop until we get a proper profile
+        final String p2ProfileId = getP2ProfileId();
+        do {
+            try {
+                Thread.sleep(50);
+            } catch (InterruptedException e) {
+                interrupted = true;
+            }
+            if (agent != null) {
+                agent.stop();
+            }
+            agent = agentProvider.createAgent(getP2AgentUri());
+            IProfileRegistry profRegistry = (IProfileRegistry) agent.getService(IProfileRegistry.SERVICE_NAME);
+            profile = profRegistry.getProfile(p2ProfileId);
+        } while (profile != null && profile.getTimestamp() == 0 && !interrupted && !progress.isCanceled());
+
+        if (profile == null || progress.isCanceled()) {
+            throw new ProvisionException("Could not find the p2 profile named " + p2ProfileId); //$NON-NLS-1$
+        }
+        return profile;
+    }
+
     /**
      * created for JUnit test so that profile Id can be changed by overriding
      *
@@ -815,4 +823,14 @@ public class P2ExtraFeature extends AbstractExtraFeature implements IP2Feature {
     public boolean useP2Cache() {
         return this.useP2Cache;
     }
+
+    protected Collection<IInstallableUnit> getInstalledP2Feature(IProfile profile, Collection<IInstallableUnit> toInstalls,
+            IProgressMonitor monitor) throws Exception {
+        Collection<IQuery<IInstallableUnit>> queries = new LinkedList<>();
+        for (IInstallableUnit toInst : toInstalls) {
+            queries.add(QueryUtil.createIUQuery(toInst.getId()));
+        }
+        return profile.available(QueryUtil.createCompoundQuery(queries, false), monitor).toSet();
+    }
+
 }

main/plugins/org.talend.updates.runtime/src/main/java/org/talend/updates/runtime/nexus/component/ComponentsDeploymentManager.java

@@ -15,6 +15,7 @@ package org.talend.updates.runtime.nexus.component;
 import java.io.File;
 import java.io.IOException;
 import java.util.Collection;
+import java.util.List;
 
 import org.apache.commons.lang3.StringUtils;
 import org.eclipse.core.runtime.IProgressMonitor;
@@ -82,7 +83,7 @@ public class ComponentsDeploymentManager {
         ITaCoKitUpdateService taCoKitService = null;
         if (GlobalServiceRegister.getDefault().isServiceRegistered(ITaCoKitUpdateService.class)) {
             try {
-                taCoKitService = (ITaCoKitUpdateService) GlobalServiceRegister.getDefault()
+                taCoKitService = GlobalServiceRegister.getDefault()
                         .getService(ITaCoKitUpdateService.class);
                 isCar = taCoKitService.isCar(componentFile, progress);
             } catch (Exception e) {
@@ -128,6 +129,11 @@ public class ComponentsDeploymentManager {
             return false;
         }
         try {
+            List<MavenArtifact> search = handler.search(mvnArtifact.getGroupId(), mvnArtifact.getArtifactId(),
+                    mvnArtifact.getVersion(), true, true);
+            if (search != null && search.size() > 0) {
+                return false;
+            }
             handler.deploy(componentFile, mvnArtifact.getGroupId(), mvnArtifact.getArtifactId(), mvnArtifact.getClassifier(),
                     mvnArtifact.getType(), mvnArtifact.getVersion());
 
@@ -174,7 +180,6 @@ public class ComponentsDeploymentManager {
             handler.deploy(indexFile, indexArtifact.getGroupId(), indexArtifact.getArtifactId(), indexArtifact.getClassifier(),
                     indexArtifact.getType(), indexArtifact.getVersion());
 
-            moveToSharedFolder(componentFile);
             return true;
         } catch (Exception e) {
             ExceptionHandler.process(e);

test/plugins/org.talend.commons.runtime.test/src/org/talend/commons/utils/resource/UpdatesHelperTest.java

@@ -162,4 +162,12 @@ public class UpdatesHelperTest {
         compDefineFile.createNewFile();
         Assert.assertTrue(UpdatesHelper.isOldComponent(compDefineFolder));
     }
+
+    @Test
+    public void test_isOldComponent_templates() throws IOException {
+        File compDefineFolder = new File(workFolder, UpdatesHelper.COMPONENT_TEMPLATES);
+        compDefineFolder.mkdirs();
+        Assert.assertTrue(UpdatesHelper.isOldComponent(compDefineFolder));
+    }
+
 }

test/plugins/org.talend.core.test/src/org/talend/designer/runprocess/ProcessorUtilitiesTest.java

@@ -15,6 +15,7 @@ package org.talend.designer.runprocess;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.anyObject;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -22,12 +23,15 @@ import static org.mockito.Mockito.when;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import org.eclipse.core.runtime.Path;
 import org.junit.Before;
 import org.junit.Test;
+import org.talend.commons.exception.PersistenceException;
 import org.talend.core.model.components.EComponentType;
 import org.talend.core.model.components.IComponent;
 import org.talend.core.model.general.ModuleNeeded;
@@ -39,8 +43,21 @@ import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.INode;
 import org.talend.core.model.process.IProcess;
 import org.talend.core.model.process.JobInfo;
+import org.talend.core.model.process.ProcessUtils;
+import org.talend.core.model.properties.ProcessItem;
+import org.talend.core.model.properties.PropertiesFactory;
+import org.talend.core.model.properties.Property;
+import org.talend.core.model.relationship.Relation;
+import org.talend.core.model.relationship.RelationshipItemBuilder;
+import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.runprocess.shadow.ObjectElementParameter;
+import org.talend.core.repository.model.ProxyRepositoryFactory;
 import org.talend.core.runtime.process.LastGenerationInfo;
+import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
+import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
+import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
+import org.talend.designer.core.model.utils.emf.talendfile.TalendFileFactory;
+import org.talend.repository.ProjectManager;
 
 /**
  * DOC ggu class global comment. Detailled comment
@@ -436,4 +453,101 @@ public class ProcessorUtilitiesTest {
         when(column2.getTalendType()).thenReturn("id_Dynamic");
         assertTrue("DB node in job. shoud be metadata dynamic", ProcessorUtilities.hasMetadataDynamic(proc, null));
     }
+
+    @Test
+    public void testCheckLoopDependencies() {
+        String projectTecLabel = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
+        ProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
+        RelationshipItemBuilder relationshipItemBuilder = RelationshipItemBuilder.getInstance();
+
+        ProcessItem item = prepareProcessItem(factory.getNextId(), "test", "0.1");
+        ProcessItem item1 = prepareProcessItem(factory.getNextId(), "test1", "0.1");
+        ProcessItem item2 = prepareProcessItem(factory.getNextId(), "test2", "0.1");
+        prepareTRunjobNode(item, projectTecLabel, item1.getProperty().getId(), RelationshipItemBuilder.LATEST_VERSION);
+        prepareTRunjobNode(item1, projectTecLabel, item2.getProperty().getId(), RelationshipItemBuilder.LATEST_VERSION);
+        prepareTRunjobNode(item2, projectTecLabel, item.getProperty().getId(), RelationshipItemBuilder.LATEST_VERSION);
+
+        IRepositoryViewObject repositoryObject = null;
+        IRepositoryViewObject repositoryObject1 = null;
+        IRepositoryViewObject repositoryObject2 = null;
+        try {
+            factory.create(item, new Path(""));
+            factory.create(item1, new Path(""));
+            factory.create(item2, new Path(""));
+
+            repositoryObject = factory.getSpecificVersion(item.getProperty().getId(), item.getProperty().getVersion(), true);
+            repositoryObject1 = factory.getSpecificVersion(item1.getProperty().getId(), item1.getProperty().getVersion(), true);
+            repositoryObject2 = factory.getSpecificVersion(item2.getProperty().getId(), item2.getProperty().getVersion(), true);
+            relationshipItemBuilder.addOrUpdateItem(repositoryObject.getProperty().getItem());
+            relationshipItemBuilder.addOrUpdateItem(repositoryObject1.getProperty().getItem());
+            relationshipItemBuilder.addOrUpdateItem(repositoryObject2.getProperty().getItem());
+
+            Relation mainRelation = new Relation();
+            mainRelation.setId(repositoryObject.getProperty().getId());
+            mainRelation.setVersion(repositoryObject.getProperty().getVersion());
+            mainRelation.setType(RelationshipItemBuilder.JOB_RELATION);
+            // job-->job1-->job2-->job hasLoop==true
+            boolean hasLoop = ProcessorUtilities.checkLoopDependencies(mainRelation, new HashMap<String, String>());
+            assertTrue(hasLoop);
+
+            // job-->job1-->job2-->job(tRunjob deactivate)
+            ProcessItem jobItem2 = (ProcessItem) repositoryObject2.getProperty().getItem();
+            for (Object nodeObject : jobItem2.getProcess().getNode()) {
+                NodeType node = (NodeType) nodeObject;
+                if (!node.getComponentName().equals("tRunJob")) { // $NON-NLS-1$
+                    continue;
+                }
+                ElementParameterType actParam = TalendFileFactory.eINSTANCE.createElementParameterType();
+                actParam.setField("CHECK");
+                actParam.setName("ACTIVATE");
+                actParam.setValue("false");
+                node.getElementParameter().add(actParam);
+            }
+            factory.save(jobItem2, false);
+            hasLoop = ProcessorUtilities.checkLoopDependencies(mainRelation, new HashMap<String, String>());
+            assertFalse(hasLoop);
+
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail("Test CheckLoopDependencies failure.");
+        } finally {
+            try {
+                factory.deleteObjectPhysical(repositoryObject);
+                factory.deleteObjectPhysical(repositoryObject1);
+                factory.deleteObjectPhysical(repositoryObject2);
+            } catch (PersistenceException e) {
+                e.printStackTrace();
+                fail("Test CheckLoopDependencies failure.");
+            }
+        }
+    }
+
+    private ProcessItem prepareProcessItem(String id, String label, String version) {
+        Property property = PropertiesFactory.eINSTANCE.createProperty();
+        ProcessItem item = PropertiesFactory.eINSTANCE.createProcessItem();
+        ProcessType process = TalendFileFactory.eINSTANCE.createProcessType();
+        item.setProperty(property);
+        item.setProcess(process);
+        property.setId(id);
+        property.setLabel(label);
+        property.setVersion(version);
+        return item;
+    }
+
+    private void prepareTRunjobNode(ProcessItem item, String projectLabel, String subjobId, String subjobVersion) {
+        NodeType node = TalendFileFactory.eINSTANCE.createNodeType();
+        node.setComponentName("tRunJob");
+        item.getProcess().getNode().add(node);
+        ElementParameterType versionParam = TalendFileFactory.eINSTANCE.createElementParameterType();
+        versionParam.setField("TECHNICAL");
+        versionParam.setName("PROCESS:PROCESS_TYPE_VERSION");
+        versionParam.setValue(subjobVersion);
+        node.getElementParameter().add(versionParam);
+        ElementParameterType jobIdParam = TalendFileFactory.eINSTANCE.createElementParameterType();
+        jobIdParam.setField("TECHNICAL");
+        jobIdParam.setName("PROCESS:PROCESS_TYPE_PROCESS");
+        jobIdParam.setValue(projectLabel + ProcessUtils.PROJECT_ID_SEPARATOR + subjobId);
+        node.getElementParameter().add(jobIdParam);
+    }
+
 }

test/plugins/org.talend.datatools.xml.test/resources/simple_node_test.xsd

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
+
+<xs:element name="day" type="Day"/>
+<xs:simpleType name="Day">
+   <xs:restriction base="xs:unsignedByte">
+      <xs:minInclusive value="1"/>
+      <xs:maxInclusive value="31"/>
+   </xs:restriction>
+</xs:simpleType>
+
+<xs:element name="shiporder">
+  <xs:complexType>
+    <xs:sequence>
+      <xs:element name="orderperson" type="xs:string"/>
+      <xs:element name="shipto">
+        <xs:complexType>
+          <xs:sequence>
+            <xs:element name="name" type="xs:string"/>
+            <xs:element name="address" type="xs:string"/>
+            <xs:element name="city" type="xs:string"/>
+            <xs:element name="country" type="xs:string"/>
+          </xs:sequence>
+        </xs:complexType>
+      </xs:element>
+      <xs:element name="item" maxOccurs="unbounded">
+        <xs:complexType>
+          <xs:sequence>
+            <xs:element name="title" type="xs:string"/>
+            <xs:element name="note" type="xs:string" minOccurs="0"/>
+            <xs:element name="quantity" type="xs:positiveInteger"/>
+            <xs:element name="price" type="xs:decimal"/>
+          </xs:sequence>
+        </xs:complexType>
+      </xs:element>
+      <xs:element name="myList">
+      	<xs:simpleType>
+      		<xs:list itemType="xs:string"/>
+      	</xs:simpleType>
+      </xs:element>
+    </xs:sequence>
+    <xs:attribute name="orderid" type="xs:string" use="required"/>
+  </xs:complexType>
+</xs:element>
+
+</xs:schema>

test/plugins/org.talend.datatools.xml.test/src/main/java/org/talend/datatools/xml/utils/XSDPopulationUtil2Test.java

@@ -115,6 +115,46 @@ public class XSDPopulationUtil2Test {
 
     }
 
+    @Test
+    public void testSimpleNode_list() throws URISyntaxException, IOException, OdaException {
+        URL resource = this.getClass().getClassLoader().getResource("resources/simple_node_test.xsd");
+        File testFile = new File(FileLocator.toFileURL(resource).toURI());
+
+        XSDPopulationUtil2 populate = new XSDPopulationUtil2();
+        XSDSchema xsdSchema = populate.getXSDSchema(testFile.getAbsolutePath());
+        List<ATreeNode> allRootNodes = populate.getAllRootNodes(xsdSchema);
+        Assert.assertEquals(allRootNodes.size(), 2);
+
+        ATreeNode shiporder = allRootNodes.get(1);
+        shiporder = populate.getSchemaTree(xsdSchema, shiporder);
+        Assert.assertEquals(shiporder.getLabel(), "shiporder");
+
+        Object[] children = shiporder.getChildren();
+        Assert.assertEquals(children.length, 5);
+        ATreeNode myListNode = ((ATreeNode) children[4]);
+        Assert.assertEquals(myListNode.getLabel(), "myList");
+        Assert.assertEquals(myListNode.getDataType(), "xs:list");
+    }
+
+    @Test
+    public void testSimpleNode_root() throws URISyntaxException, IOException, OdaException {
+        URL resource = this.getClass().getClassLoader().getResource("resources/simple_node_test.xsd");
+        File testFile = new File(FileLocator.toFileURL(resource).toURI());
+
+        XSDPopulationUtil2 populate = new XSDPopulationUtil2();
+        XSDSchema xsdSchema = populate.getXSDSchema(testFile.getAbsolutePath());
+        List<ATreeNode> allRootNodes = populate.getAllRootNodes(xsdSchema);
+        Assert.assertEquals(allRootNodes.size(), 2);
+
+        ATreeNode day = allRootNodes.get(0);
+        day = populate.getSchemaTree(xsdSchema, day);
+        Assert.assertEquals(day.getLabel(), "day");
+
+        Object[] children = day.getChildren();
+        Assert.assertEquals(children.length, 0);
+        Assert.assertEquals(day.getDataType(), "xs:unsignedByte");
+    }
+
     private void testBranches(ATreeNode testBranchNode) {
         Object[] children = testBranchNode.getChildren();
         Assert.assertEquals(children.length, 1);

test/plugins/org.talend.librariesmanager.test/src/org/talend/librariesmanager/model/ModulesNeededProviderTest.java

@@ -35,6 +35,7 @@ public class ModulesNeededProviderTest {
     public void testUpdateModulesNeededForRoutine() throws Exception {
         String jarName1 = "testUpdateModulesNeededForRoutine1.jar";
         String jarName2 = "testUpdateModulesNeededForRoutine2.jar";
+        String jarName3 = "testUpdateModulesNeededForRoutine3.jar";
 
         String message = ModuleNeeded.UNKNOWN;
 
@@ -43,20 +44,26 @@ public class ModulesNeededProviderTest {
         routineItem.getProperty().setLabel("routineTest");
         IMPORTType importJar1 = ComponentFactory.eINSTANCE.createIMPORTType();
         importJar1.setMODULE(jarName1);
+        importJar1.setREQUIRED(true);
         importJar1.setNAME(routineItem.getProperty().getLabel());
         routineItem.getImports().add(importJar1);
         IMPORTType importJar2 = ComponentFactory.eINSTANCE.createIMPORTType();
         importJar2.setMODULE(jarName2);
+        importJar2.setREQUIRED(true);
         importJar2.setNAME(routineItem.getProperty().getLabel());
         routineItem.getImports().add(importJar2);
-
+        IMPORTType importJar3 = ComponentFactory.eINSTANCE.createIMPORTType();
+        importJar3.setMODULE(jarName3);
+        importJar3.setREQUIRED(false);
+        importJar3.setNAME(routineItem.getProperty().getLabel());
+        routineItem.getImports().add(importJar3);
         // remove old modules from the two lists before test
         String label = ERepositoryObjectType.getItemType(routineItem).getLabel();
         String currentContext = label + " " + routineItem.getProperty().getLabel();
         Set<ModuleNeeded> oldModules = new HashSet<ModuleNeeded>();
         for (ModuleNeeded existingModule : ModulesNeededProvider.getModulesNeeded()) {
             if (currentContext.equals(existingModule.getContext()) || jarName1.equals(existingModule.getModuleName())
-                    || jarName2.equals(existingModule.getModuleName())) {
+                    || jarName2.equals(existingModule.getModuleName()) || jarName3.equals(existingModule.getModuleName())) {
                 oldModules.add(existingModule);
             }
         }
@@ -64,7 +71,7 @@ public class ModulesNeededProviderTest {
         oldModules = new HashSet<ModuleNeeded>();
         for (ModuleNeeded existingModule : ModulesNeededProvider.getAllManagedModules()) {
             if (currentContext.equals(existingModule.getContext()) || jarName1.equals(existingModule.getModuleName())
-                    || jarName2.equals(existingModule.getModuleName())) {
+                    || jarName2.equals(existingModule.getModuleName()) || jarName3.equals(existingModule.getModuleName())) {
                 oldModules.add(existingModule);
             }
         }
@@ -75,7 +82,7 @@ public class ModulesNeededProviderTest {
         int originalAllSize = ModulesNeededProvider.getAllManagedModules().size();
 
         ModulesNeededProvider.updateModulesNeededForRoutine(routineItem);
-        // add two modules to needed list
+        // add 3 modules to needed list, but one of them is require false
         Assert.assertEquals(ModulesNeededProvider.getModulesNeeded().size(), originalNeededSize + 2);
         // add one + change one in the all list
         Assert.assertEquals(ModulesNeededProvider.getAllManagedModules().size(), originalAllSize + 1);

test/plugins/org.talend.librariesmanager.test/src/org/talend/librariesmanager/nexus/utils/VersionUtilTest.java

@@ -0,0 +1,38 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.librariesmanager.nexus.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * DOC hwang  class global comment. Detailled comment
+ */
+public class VersionUtilTest {
+
+    @Test
+    public void testGetSNAPSHOTVersion() {
+        VersionUtil util = new VersionUtil();
+        String rVersion = "6.0.0-20191015.030844-1";
+        String result = util.getSNAPSHOTVersion(rVersion);
+        Assert.assertEquals(result, "6.0.0-SNAPSHOT");
+        
+        rVersion = null;
+        result = util.getSNAPSHOTVersion(rVersion);
+        Assert.assertNull(result);
+        
+        rVersion = "6.0.0";
+        result = util.getSNAPSHOTVersion(rVersion);
+        Assert.assertEquals(result, "6.0.0");
+    }
+}