fix(TUP-32088): check remote snapshot artifacts (#4409)

* fix(TUP-32088): check remote snapshot artifacts

* fix(TUP-32088): fix set sha1

* fix(TUP-32088): if package type is null, can't set sha1

main/plugins/org.talend.commons.runtime/META-INF/MANIFEST.MF

@@ -13,7 +13,9 @@ Require-Bundle: org.apache.log4j;visibility:=reexport,
  org.talend.libraries.apache,
  org.apache.xerces,
  com.ibm.icu,
- org.apache.commons.io
+ org.apache.commons.io,
+ org.talend.utils,
+ org.eclipse.core.net
 Export-Package: org.talend.commons,
  org.talend.commons.exception,
  org.talend.commons.i18n,

main/plugins/org.talend.commons.runtime/src/org/talend/commons/CommonsPlugin.java

@@ -17,11 +17,13 @@ import java.io.InputStream;
 import java.net.URL;
 
 import org.apache.commons.lang.ArrayUtils;
+import org.eclipse.core.net.proxy.IProxyService;
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.osgi.framework.BundleActivator;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.ServiceReference;
+import org.osgi.util.tracker.ServiceTracker;
 import org.talend.commons.exception.ExceptionService;
 import org.talend.commons.runtime.debug.TalendDebugHandler;
 
@@ -52,6 +54,8 @@ public class CommonsPlugin implements BundleActivator {
     // TESB-17856: For commandline builds ESB Micorservice bundle
     private static boolean isESBMicorservice = false;
 
+    private static ServiceTracker proxyTracker;
+
     public static boolean isWorkbenchCreated() {
         return isWorkbenchCreated;
     }
@@ -160,4 +164,13 @@ public class CommonsPlugin implements BundleActivator {
     	InstanceScope.INSTANCE.getNode("org.eclipse.m2e.core").putBoolean("eclipse.m2.offline", state);
     }
 
+    public static IProxyService getProxyService() {
+        if (proxyTracker == null) {
+            proxyTracker = new ServiceTracker(Platform.getBundle(PLUGIN_ID).getBundleContext(), IProxyService.class.getName(),
+                    null);
+            proxyTracker.open();
+        }
+        return (IProxyService) proxyTracker.getService();
+    }
+
 }

main/plugins/org.talend.commons.runtime/src/org/talend/commons/exception/ClientException.java

@@ -0,0 +1,58 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.commons.exception;
+
+public class ClientException extends PersistenceException {
+    
+    private Integer httpCode;
+
+    public ClientException(String message) {
+        super(message);
+    }
+
+    public ClientException(Integer httpCode, String message) {
+        super(message);
+        this.httpCode = httpCode;
+    }
+
+    public ClientException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public ClientException(Integer httpCode, String message, Throwable cause) {
+        super(message, cause);
+        this.httpCode = httpCode;
+    }
+
+    public ClientException(Throwable cause) {
+        super(cause);
+    }
+
+    public ClientException(Integer httpCode, Throwable cause) {
+        super(cause);
+        this.httpCode = httpCode;
+    }
+
+    public Integer getHttpCode() {
+        return httpCode;
+    }
+
+    public void setHttpCode(Integer httpCode) {
+        this.httpCode = httpCode;
+    }
+
+    @Override
+    public String toString() {
+        return getLocalizedMessage();
+    }
+}

main/plugins/org.talend.commons.runtime/src/org/talend/commons/i18n/internal/messages.properties

@@ -58,3 +58,6 @@ AS400ResultSet.unknowCloumn=Invalid argument\: unknown column name
 AS400ResultSet.parameterIndex=Invalid argument\: parameter index
 AS400ResultSet.outofRange=\ is out of range.
 ITaCoKitService.exception.multipleInstance=More than one instance found: {0}
+TalendProxySelector.exception.badUriMap=Bad uri map: {0}
+TalendProxySelector.exception.proxySelectionError=Error occurs when selecting proxy for {0}
+

main/plugins/org.talend.commons.runtime/src/org/talend/commons/runtime/service/ITaCoKitService.java

@@ -35,6 +35,8 @@ public interface ITaCoKitService {
 
     boolean isTaCoKitCar(File file, IProgressMonitor monitor) throws Exception;
 
+    boolean isNeedMigration(String projectLabel) throws Exception;
+
     void checkMigration(final IProgressMonitor monitor) throws Exception;
 
     public static ITaCoKitService getInstance() throws Exception {

main/plugins/org.talend.commons.runtime/src/org/talend/commons/runtime/utils/io/FileCopyUtils.java

@@ -123,6 +123,10 @@ public class FileCopyUtils {
         if (!resFolder.exists()) {
             return;
         }
+        if (resFolder.getAbsolutePath().equals(destFolder.getAbsolutePath())) {// If the source path and target path are
+                                                                               // same, don't copy
+            return;
+        }
         destFolder.mkdirs();
         String[] file = resFolder.list();
         File temp = null;

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/database/SAPHanaDataBaseMetadata.java

@@ -120,33 +120,41 @@ public class SAPHanaDataBaseMetadata extends FakeDatabaseMetaData {
     public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types)
             throws SQLException {
         String[] neededTypes = getNeededTypes(types);
-        ResultSet rs = connection.getMetaData().getTables(catalog, schemaPattern, tableNamePattern, neededTypes);
         List<String[]> list = new ArrayList<String[]>();
-        while (rs.next()) {
-            String name = rs.getString("TABLE_NAME"); //$NON-NLS-1$
-            String schema = rs.getString("TABLE_SCHEM"); //$NON-NLS-1$
-            String type = rs.getString("TABLE_TYPE"); //$NON-NLS-1$
+        // has other types which is not Calculation View
+        if (neededTypes.length > 1 || !ArrayUtils.contains(neededTypes, NEEDED_TYPES[3])) {
+            ResultSet rs = connection.getMetaData().getTables(catalog, schemaPattern, tableNamePattern, neededTypes);
+            while (rs.next()) {
+                String name = rs.getString("TABLE_NAME"); //$NON-NLS-1$
+                String schema = rs.getString("TABLE_SCHEM"); //$NON-NLS-1$
+                String type = rs.getString("TABLE_TYPE"); //$NON-NLS-1$
 
-            String id = ""; //$NON-NLS-1$
-            String remarks = ""; //$NON-NLS-1$
-            try {
-                remarks = rs.getString("REMARKS"); //$NON-NLS-1$
-            } catch (Exception e) {
-                // nothing
-            }
+                String id = ""; //$NON-NLS-1$
+                String remarks = ""; //$NON-NLS-1$
+                try {
+                    remarks = rs.getString("REMARKS"); //$NON-NLS-1$
+                } catch (Exception e) {
+                    // nothing
+                }
 
-            if (ArrayUtils.contains(neededTypes, type)) {
-                // check if the type is contained is in the types needed.
-                // since sybase can return some system views as "SYSTEM VIEW" instead of "VIEW/TABLE" from the request.
-                String[] r = new String[] { id, schema, name, type, remarks, null };
-                list.add(r);
+                if (ArrayUtils.contains(neededTypes, type)) {
+                    // check if the type is contained is in the types needed.
+                    // since sybase can return some system views as "SYSTEM VIEW" instead of "VIEW/TABLE" from the
+                    // request.
+                    String[] r = new String[] { id, schema, name, type, remarks, null };
+                    list.add(r);
+                }
             }
         }
 
         // For Calculation View
         if (ArrayUtils.contains(neededTypes, NEEDED_TYPES[3])) {
             // check if the type is contained is in the types needed.
-            String sqlcv = "SELECT CATALOG_NAME,SCHEMA_NAME,CUBE_NAME, COLUMN_OBJECT,CUBE_TYPE,DESCRIPTION from _SYS_BI.BIMC_CUBES"; //$NON-NLS-1$
+            String sqlcv = "SELECT OBJECT_NAME,PACKAGE_ID FROM _SYS_REPO.ACTIVE_OBJECT WHERE OBJECT_SUFFIX = 'calculationview'"; //$NON-NLS-1$
+            if (tableNamePattern != null && !tableNamePattern.equals("%")) { //$NON-NLS-1$
+                sqlcv += " AND (OBJECT_NAME LIKE '" + tableNamePattern + "'"; //$NON-NLS-1$ //$NON-NLS-2$
+                sqlcv += " OR PACKAGE_ID LIKE '" + tableNamePattern + "')"; //$NON-NLS-1$ //$NON-NLS-2$
+            }
             ResultSet rscv = null;
             Statement stmtcv = null;
             List<String[]> listcv = new ArrayList<String[]>();
@@ -154,25 +162,16 @@ public class SAPHanaDataBaseMetadata extends FakeDatabaseMetaData {
                 stmtcv = connection.createStatement();
                 rscv = stmtcv.executeQuery(sqlcv);
                 while (rscv.next()) {
-                    String catalogName = rscv.getString("CATALOG_NAME"); //$NON-NLS-1$
-                    if (catalogName != null) {
-                        catalogName = catalogName.trim();
+                    String objectName = rscv.getString("OBJECT_NAME"); //$NON-NLS-1$
+                    if (objectName != null) {
+                        objectName = objectName.trim();
                     }
-                    String schemaName = rscv.getString("SCHEMA_NAME"); //$NON-NLS-1$
-                    if (schemaName != null) {
-                        schemaName = schemaName.trim();
+                    String packageId = rscv.getString("PACKAGE_ID"); //$NON-NLS-1$
+                    if (packageId != null) {
+                        packageId = packageId.trim();
                     }
-                    String cubeName = rscv.getString("CUBE_NAME"); //$NON-NLS-1$
-                    if (cubeName != null) {
-                        cubeName = cubeName.trim();
-                    }
-                    String id = ""; //$NON-NLS-1$
-                    // String type = rscv.getString("CUBE_TYPE"); //$NON-NLS-1$
-
-                    String remarks = rscv.getString("DESCRIPTION"); //$NON-NLS-1$
-                    String name = catalogName + "/" + cubeName;//$NON-NLS-1$
-
-                    String[] r = new String[] { id, schemaName, name, NEEDED_TYPES[3], remarks, catalogName };
+                    String name = packageId + "/" + objectName; //$NON-NLS-1$
+                    String[] r = new String[] { "", "_SYS_BIC", name, NEEDED_TYPES[3], "", packageId }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
                     listcv.add(r);
                 }
             } catch (SQLException e) {

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/network/NetworkUtil.java

@@ -42,13 +42,17 @@ public class NetworkUtil {
 
     private static final String TALEND_DISABLE_INTERNET = "talend.disable.internet";//$NON-NLS-1$
 
+    private static final String HTTP_NETWORK_URL = "https://talend-update.talend.com";
+
+    public static final String ORG_TALEND_DESIGNER_CORE = "org.talend.designer.core";
+
     public static boolean isNetworkValid() {
         String disableInternet = System.getProperty(TALEND_DISABLE_INTERNET);
         if ("true".equals(disableInternet)) { //$NON-NLS-1$
             return false;
         }
         try {
-            URL url = new URL("https://www.talend.com"); //$NON-NLS-1$
+            URL url = new URL(HTTP_NETWORK_URL);
             HttpURLConnection conn = (HttpURLConnection) url.openConnection();
             conn.setConnectTimeout(4000);
             conn.setReadTimeout(4000);

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/network/TalendProxySelector.java

@@ -13,37 +13,211 @@
 package org.talend.commons.utils.network;
 
 import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.net.PasswordAuthentication;
 import java.net.Proxy;
 import java.net.ProxySelector;
 import java.net.SocketAddress;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Priority;
+import org.eclipse.core.internal.net.ProxyManager;
+import org.eclipse.core.net.proxy.IProxyService;
+import org.talend.commons.CommonsPlugin;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.i18n.internal.Messages;
+import org.talend.daikon.sandbox.properties.ClassLoaderIsolatedSystemProperties;
+
+import sun.net.spi.DefaultProxySelector;
 
 /**
  * DOC cmeng  class global comment. Detailled comment
  */
 public class TalendProxySelector extends ProxySelector {
 
+    public static final String PROP_PRINT_LOGS = "talend.studio.proxy.printLogs";
+
     private static final String ECLIPSE_PROXY_SELECTOR = ".EclipseProxySelector"; //$NON-NLS-1$
 
-    private ProxySelector defaultSelector;
+    private static final String PROP_ALLOW_PROXY_REDIRECT = "talend.studio.proxy.allowProxyRedirect";
 
-    final private List<IProxySelectorProvider> selectorProviders;
+    private static final String PROP_ALLOW_PROXY_REDIRECT_EXCLUDE = "talend.studio.proxy.redirect.whiteList";
+
+    private static final String PROP_EXECUTE_CONNECTION_FAILED = "talend.studio.proxy.executeConnectionFailed";
+
+    private static final String PROP_UPDATE_SYSTEM_PROPERTIES_FOR_JRE = "talend.studio.proxy.jre.updateSystemProperties";
+
+    private static final String PROP_CHECK_PROXY = "talend.studio.proxy.checkProxy";
+
+    private static final String PROP_VALIDATE_URI = "talend.studio.proxy.validateUri";
+
+    private static final String PROP_PROXY_SELECTOR = "talend.studio.proxy.selector";
+
+    private static final String PROP_PROXY_SELECTOR_DEFAULT = "default";
+
+    private static final String PROP_PROXY_SELECTOR_JRE = "jre";
+
+    private static final String PROP_PROXY_HOST_MAP = "talend.studio.proxy.hostMap";
+
+    private static final String PROP_DISABLE_DEFAULT_SELECTOR_PROVIDER = "talend.studio.proxy.disableDefaultSelectorProvider";
+
+    /**
+     * Example: update.talend.com,socket:http,https:http;nexus.talend.com,socket,http;,socket:http
+     */
+    private static final String PROP_PROXY_MAP_HOST_DEFAULT = "";
+
+    /**
+     * Example: svn.company.com;nexus.company.com
+     */
+    private static final String PROP_ALLOW_PROXY_REDIRECT_EXCLUDE_DEFAULT = "";
+
+    private static final String KEY_DEFAULT = ":default:";
+
+    private static Field uriHostField;
+
+    private static Method proxyManagerUpdateSystemPropertiesFunc;
+
+    private static boolean checkProxy = Boolean.valueOf(System.getProperty(PROP_CHECK_PROXY, Boolean.TRUE.toString()));
+
+    /**
+     * Note: eclipse default selector may be different between TOS and TIS, TOS may use jre one, TIS may use egit one
+     */
+    private ProxySelector eclipseDefaultSelector;
+
+    private ProxySelector jreDefaultSelector;
+
+    private EProxySelector eProxySelector;
+
+    final private Map<Object, Collection<IProxySelectorProvider>> selectorProviders;
+
+    private Map<String, Map<String, String>> hostMap;
+
+    private Set<String> redirectWhiteList;
 
     private volatile static TalendProxySelector instance;
 
     private static Object instanceLock = new Object();
 
-    private TalendProxySelector(final ProxySelector defaultSelector) {
-        this.defaultSelector = defaultSelector;
-        selectorProviders = new ArrayList<>();
+    private boolean printProxyLog = false;
+
+    private boolean allowProxyRedirect = false;
+
+    private boolean disableDefaultSelectorProvider = false;
+
+    private boolean validateUri = true;
+
+    private boolean executeConnectionFailed = true;
+
+    private boolean updateSystemPropertiesForJre = true;
+
+    private TalendProxySelector(final ProxySelector eclipseDefaultSelector) {
+        this.eclipseDefaultSelector = eclipseDefaultSelector;
+        this.jreDefaultSelector = new DefaultProxySelector();
+
+        selectorProviders = Collections.synchronizedMap(new HashMap<>());
+        allowProxyRedirect = Boolean.valueOf(System.getProperty(PROP_ALLOW_PROXY_REDIRECT, Boolean.FALSE.toString()));
+        disableDefaultSelectorProvider = Boolean
+                .valueOf(System.getProperty(PROP_DISABLE_DEFAULT_SELECTOR_PROVIDER, Boolean.FALSE.toString()));
+        printProxyLog = Boolean.valueOf(System.getProperty(PROP_PRINT_LOGS, Boolean.FALSE.toString()));
+        validateUri = Boolean.valueOf(System.getProperty(PROP_VALIDATE_URI, Boolean.TRUE.toString()));
+        executeConnectionFailed = Boolean.valueOf(System.getProperty(PROP_EXECUTE_CONNECTION_FAILED, Boolean.TRUE.toString()));
+        updateSystemPropertiesForJre = Boolean
+                .valueOf(System.getProperty(PROP_UPDATE_SYSTEM_PROPERTIES_FOR_JRE, Boolean.TRUE.toString()));
+
+        switch (System.getProperty(PROP_PROXY_SELECTOR, PROP_PROXY_SELECTOR_DEFAULT).toLowerCase()) {
+        case PROP_PROXY_SELECTOR_JRE:
+            this.eProxySelector = EProxySelector.jre;
+            break;
+        default:
+            this.eProxySelector = EProxySelector.eclipse_default;
+            break;
+        }
+
+        initHostMap();
+        initRedirectList();
+    }
+
+    private void initHostMap() {
+        try {
+            hostMap = new HashMap<>();
+            String property = System.getProperty(PROP_PROXY_HOST_MAP, PROP_PROXY_MAP_HOST_DEFAULT);
+            if (StringUtils.isEmpty(property)) {
+                return;
+            }
+            String[] splits = property.split(";");
+            for (String split : splits) {
+                try {
+                    int index = split.indexOf(',');
+                    String uri = split.substring(0, index);
+                    String key = StringUtils.strip(uri);
+                    if (StringUtils.isBlank(key)) {
+                        key = KEY_DEFAULT;
+                    }
+                    key = key.toLowerCase();
+                    Map<String, String> protocolMap = hostMap.get(key);
+                    if (protocolMap == null) {
+                        protocolMap = new HashMap<>();
+                        hostMap.put(key, protocolMap);
+                    }
+                    int protocolMapIndex = index + 1;
+                    String protocolMapStr = split.substring(protocolMapIndex);
+                    String[] entry = protocolMapStr.split(",");
+                    for (String pMap : entry) {
+                        try {
+                            String[] mapEntry = pMap.split(":");
+                            if (mapEntry.length != 2) {
+                                ExceptionHandler.process(
+                                        new Exception(Messages.getString("TalendProxySelector.exception.badUriMap", pMap)));
+                                continue;
+                            }
+                            protocolMap.put(mapEntry[0].toLowerCase(), mapEntry[1].toLowerCase());
+                        } catch (Exception e) {
+                            ExceptionHandler.process(e);
+                        }
+                    }
+                } catch (Exception e) {
+                    ExceptionHandler.process(e);
+                }
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
+    private void initRedirectList() {
+        try {
+            redirectWhiteList = new HashSet<>();
+            String property = System.getProperty(PROP_ALLOW_PROXY_REDIRECT_EXCLUDE, PROP_ALLOW_PROXY_REDIRECT_EXCLUDE_DEFAULT);
+            if (StringUtils.isEmpty(property)) {
+                return;
+            }
+            String[] split = property.split(";");
+            for (String host : split) {
+                host = StringUtils.strip(host);
+                if (StringUtils.isBlank(host)) {
+                    host = KEY_DEFAULT;
+                }
+                redirectWhiteList.add(host);
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
     }
 
     public static TalendProxySelector getInstance() {
@@ -58,6 +232,18 @@ public class TalendProxySelector extends ProxySelector {
             synchronized (instanceLock) {
                 if (instance == null) {
                     instance = new TalendProxySelector(proxySelector);
+                    try {
+                        uriHostField = URI.class.getDeclaredField("host");
+                        uriHostField.setAccessible(true);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
+                    try {
+                        proxyManagerUpdateSystemPropertiesFunc = ProxyManager.class.getDeclaredMethod("updateSystemProperties");
+                        proxyManagerUpdateSystemPropertiesFunc.setAccessible(true);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
                 }
             }
         }
@@ -70,19 +256,216 @@ public class TalendProxySelector extends ProxySelector {
                     return null;
                 }
             });
-            if (instance.getDefaultProxySelector() == null
+            if (instance.getEclipseDefaultSelector() == null
                     || (proxySelector != null && proxySelector.getClass().getName().endsWith(ECLIPSE_PROXY_SELECTOR))) {
-                instance.setDefaultProxySelector(proxySelector);
+                instance.setEclipseDefaultSelector(proxySelector);
             }
         }
 
         return instance;
     }
 
+    public static void checkProxy() {
+        if (!checkProxy) {
+            return;
+        }
+        try {
+            TalendProxySelector.getInstance();
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
     @Override
     public List<Proxy> select(final URI uri) {
-        final Set<Proxy> resultFromProviders = new HashSet<>();
-        List<IProxySelectorProvider> providers = getProxySelectorProviders();
+        if (printProxyLog) {
+            ExceptionHandler.log("TalendProxySelector.select " + uri);
+        }
+        if (uri == null) {
+            return Collections.EMPTY_LIST;
+        }
+        URI validatedUri = validateUri(uri);
+        Set<Proxy> results = new LinkedHashSet<>();
+
+        try {
+            final Set<Proxy> resultFromProviders = getProxysFromProviders(validatedUri);
+            if (resultFromProviders != null && !resultFromProviders.isEmpty()) {
+                results.addAll(resultFromProviders);
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+        if (printProxyLog) {
+            ExceptionHandler.log("TalendProxySelector.resultFromProviders " + results);
+        }
+
+        ProxySelector defaultProxySelector = getDefaultProxySelector();
+        if (printProxyLog) {
+            ExceptionHandler.log("TalendProxySelector.defaultProxySelector " + defaultProxySelector);
+        }
+        if (defaultProxySelector != null) {
+            /**
+             * don't validate uri here, so that we can know whether it is an issue uri
+             */
+            URI newUri = getNewUri(validatedUri, false);
+            List<Proxy> defaultProxys = null;
+            if (validateUri && StringUtils.isBlank(newUri.getHost())) {
+                /**
+                 * If host is blank, force to use jre proxy selector to avoid the eclipse proxy selector bug
+                 */
+                defaultProxys = getJreProxySelector().select(newUri);
+            } else {
+                defaultProxys = defaultProxySelector.select(newUri);
+            }
+            if (printProxyLog) {
+                ExceptionHandler.log("TalendProxySelector.defaultProxys " + defaultProxys);
+            }
+            try {
+                results.addAll(filterProxys(validatedUri, defaultProxys));
+            } catch (Exception e) {
+                results.addAll(defaultProxys);
+                ExceptionHandler.process(e);
+            }
+        }
+        if (printProxyLog) {
+            String proxys = results.toString();
+            ExceptionHandler.log("Selected proxys for " + uri + ", " + proxys);
+            ExceptionHandler.process(new Exception("Proxy call stacks"), Priority.INFO);
+        }
+        return new LinkedList<Proxy>(results);
+    }
+
+    private URI validateUri(URI uri) {
+        if (!validateUri) {
+            return uri;
+        }
+
+        URI validatedUri = null;
+        try {
+            /**
+             * DON'T use URI.create(), MUST use the conductor which requires authority
+             */
+            validatedUri = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), uri.getQuery(), uri.getFragment());
+
+            /**
+             * Validate the host, if the host is empty, it will cause the eclipse selector to return dirrect
+             */
+            if (StringUtils.isBlank(validatedUri.getHost())) {
+                String authority = validatedUri.getAuthority();
+                if (StringUtils.isNotBlank(authority)) {
+                    // example: https://u:p@www.company.com:8081/path/a?param=b
+                    String host = null;
+                    int userInfoIndex = authority.indexOf('@');
+                    if (0 <= userInfoIndex) {
+                        authority = authority.substring(userInfoIndex + 1);
+                    }
+                    int portIndex = authority.lastIndexOf(':');
+                    if (0 <= portIndex) {
+                        host = authority.substring(0, portIndex);
+                    }
+                    try {
+                        uriHostField.set(validatedUri, host);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
+                }
+
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+            if (validatedUri == null) {
+                validatedUri = uri;
+            }
+        }
+        if (printProxyLog) {
+            ExceptionHandler.log("After validate: " + uri + " -> " + validatedUri);
+        }
+        return validatedUri;
+    }
+
+    private List<Proxy> filterProxys(final URI uri, List<Proxy> defaultProxys) {
+        List<Proxy> result = new ArrayList<>();
+        if (defaultProxys != null && !defaultProxys.isEmpty()) {
+            for (Proxy proxy : defaultProxys) {
+                SocketAddress addr = null;
+                Proxy.Type proxyType = null;
+                if (proxy != null) {
+                    proxyType = proxy.type();
+                    addr = proxy.address();
+                }
+
+                boolean redirect = true;
+                if (!allowProxyRedirect) {
+                    String host = uri.getHost();
+                    if (host == null) {
+                        host = "";
+                    }
+                    host = StringUtils.strip(host).toLowerCase();
+                    if (this.redirectWhiteList.contains(host) || this.redirectWhiteList.contains(KEY_DEFAULT)) {
+                        redirect = true;
+                    } else if (Proxy.Type.DIRECT == proxyType
+                            || (addr != null && StringUtils.equals(uri.getHost(), ((InetSocketAddress) addr).getHostString()))) {
+                        redirect = false;
+                    }
+                }
+                if (redirect) {
+                    result.add(proxy);
+                } else {
+                    result.add(Proxy.NO_PROXY);
+                }
+            }
+        }
+        return result;
+    }
+
+    private URI getNewUri(URI uri, boolean validateUri) {
+        URI newUri = uri;
+        if (newUri != null) {
+            try {
+                // get host before new URI, because the host may be set manually due to URI issue
+                String host = newUri.getHost();
+                newUri = new URI(newUri.getScheme(), newUri.getAuthority(), newUri.getPath(), newUri.getQuery(),
+                        newUri.getFragment());
+                Map<String, String> protocolMap = null;
+                if (StringUtils.isNotBlank(host)) {
+                    protocolMap = hostMap.get(host.toLowerCase());
+                }
+                if (protocolMap == null) {
+                    protocolMap = hostMap.get(KEY_DEFAULT);
+                }
+
+                if (protocolMap != null) {
+                    String schema = newUri.getScheme();
+                    if (schema != null) {
+                        String lowercasedProtocol = schema.toLowerCase();
+                        String preferedProtocol = protocolMap.get(lowercasedProtocol);
+                        if (StringUtils.isNotBlank(preferedProtocol)) {
+                            /**
+                             * Note: MUST use the constructor which requires authority, because some uri may be illegal,
+                             * then host info will be stored in authority field instead of host filed
+                             */
+                            newUri = new URI(preferedProtocol, newUri.getAuthority(), newUri.getPath(), newUri.getQuery(),
+                                    newUri.getFragment());
+                        }
+                    }
+                }
+            } catch (URISyntaxException e) {
+                if (printProxyLog) {
+                    ExceptionHandler.process(
+                            new Exception(Messages.getString("TalendProxySelector.exception.proxySelectionError", uri), e),
+                            Priority.WARN);
+                }
+            }
+        }
+        if (validateUri) {
+            newUri = validateUri(newUri);
+        }
+        return newUri;
+    }
+
+    private Set<Proxy> getProxysFromProviders(final URI uri) {
+        final Set<Proxy> resultFromProviders = new LinkedHashSet<>();
+        Collection<IProxySelectorProvider> providers = getCustomProviders(uri);
         if (providers != null) {
             providers.stream().forEach(p -> {
                 if (instance == p) {
@@ -100,49 +483,126 @@ public class TalendProxySelector extends ProxySelector {
                 }
             });
         }
-        List<Proxy> result = new ArrayList<>();
+        return resultFromProviders;
+    }
 
-        if (resultFromProviders != null && !resultFromProviders.isEmpty()) {
-            result.addAll(resultFromProviders);
-        }
-
-        ProxySelector defaultProxySelector = getDefaultProxySelector();
-        if (defaultProxySelector != null) {
-            List<Proxy> defaultProxys = defaultProxySelector.select(uri);
-            if (defaultProxys != null && !defaultProxys.isEmpty()) {
-                result.addAll(defaultProxys);
+    private Collection<IProxySelectorProvider> getCustomProviders(final URI uri) {
+        Collection<IProxySelectorProvider> providers = Collections.EMPTY_LIST;
+        Collection<Object> possibleKeys = getPossibleKeys(uri);
+        for (Object key : possibleKeys) {
+            providers = this.selectorProviders.get(key);
+            if (providers != null) {
+                break;
             }
         }
-        return result;
+        return providers;
+    }
+
+    public PasswordAuthentication getHttpPasswordAuthentication() {
+        String[] schemas = new String[] { "http", "https" };
+        for (String schema : schemas) {
+            String proxyUser = System.getProperty(schema + ".proxyUser");
+            String proxyPassword = System.getProperty(schema + ".proxyPassword");
+
+            if (StringUtils.isNotBlank(proxyUser)) {
+                char[] pwdChars = new char[0];
+                if (proxyPassword != null && !proxyPassword.isEmpty()) {
+                    pwdChars = proxyPassword.toCharArray();
+                }
+                return new PasswordAuthentication(proxyUser, pwdChars);
+            }
+        }
+        return null;
     }
 
     public boolean addProxySelectorProvider(IProxySelectorProvider provider) {
-        List<IProxySelectorProvider> proxySelectorProviders = getProxySelectorProviders();
-        if (!proxySelectorProviders.contains(provider)) {
-            return proxySelectorProviders.add(provider);
+        try {
+            Object key = provider.getKey();
+            Collection<IProxySelectorProvider> collection = this.selectorProviders.get(key);
+            if (collection == null) {
+                synchronized (this.selectorProviders) {
+                    collection = this.selectorProviders.get(key);
+                    if (collection == null) {
+                        collection = Collections.synchronizedList(new LinkedList<>());
+                        this.selectorProviders.put(key, collection);
+                    }
+                }
+            }
+            collection.add(provider);
+            return true;
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
         }
         return false;
     }
 
     public boolean removeProxySelectorProvider(IProxySelectorProvider provider) {
-        return getProxySelectorProviders().remove(provider);
-    }
-
-    private List<IProxySelectorProvider> getProxySelectorProviders() {
-        return selectorProviders;
+        try {
+            Object key = provider.getKey();
+            Collection<IProxySelectorProvider> collection = this.selectorProviders.get(key);
+            if (collection != null) {
+                synchronized (this.selectorProviders) {
+                    collection = this.selectorProviders.get(key);
+                    if (collection != null) {
+                        collection.remove(provider);
+                        if (collection.isEmpty()) {
+                            this.selectorProviders.remove(key);
+                        }
+                    }
+                }
+            }
+            return true;
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+        return false;
     }
 
     public ProxySelector getDefaultProxySelector() {
-        return defaultSelector;
+        switch (eProxySelector) {
+        case jre:
+            return getJreProxySelector();
+        default:
+            return eclipseDefaultSelector;
+        }
     }
 
-    public void setDefaultProxySelector(final ProxySelector selector) {
-        defaultSelector = selector;
+    private ProxySelector getJreProxySelector() {
+        try {
+            /**
+             * for tcompv0, daikon may create an isolated system properties for it, so proxies may be ignored in the new
+             * system properties; here we try to call the method to add proxies into the isolated system properties
+             */
+            if (updateSystemPropertiesForJre && ClassLoaderIsolatedSystemProperties.getInstance()
+                    .isIsolated(Thread.currentThread().getContextClassLoader())) {
+                if (printProxyLog) {
+                    ExceptionHandler.log("Before update jre proxy system properties for the isolated classloader, http.proxyHost="
+                            + System.getProperty("http.proxyHost"));
+                }
+                IProxyService proxyService = CommonsPlugin.getProxyService();
+                proxyManagerUpdateSystemPropertiesFunc.invoke(proxyService);
+                if (printProxyLog) {
+                    ExceptionHandler.log("After updated jre proxy system properties for the isolated classloader, http.proxyHost="
+                            + System.getProperty("http.proxyHost"));
+                }
+            }
+        } catch (Throwable e) {
+            ExceptionHandler.process(e);
+        }
+        return this.jreDefaultSelector;
+    }
+
+    public ProxySelector getEclipseDefaultSelector() {
+        return eclipseDefaultSelector;
+    }
+
+    public void setEclipseDefaultSelector(ProxySelector eclipseDefaultSelector) {
+        this.eclipseDefaultSelector = eclipseDefaultSelector;
     }
 
     @Override
     public void connectFailed(final URI uri, final SocketAddress sa, final IOException ioe) {
-        List<IProxySelectorProvider> providers = getProxySelectorProviders();
+        Collection<IProxySelectorProvider> providers = getCustomProviders(uri);
         if (providers != null) {
             providers.stream().forEach(p -> {
                 if (p.canHandle(uri)) {
@@ -151,23 +611,35 @@ public class TalendProxySelector extends ProxySelector {
             });
         }
 
-        ProxySelector defaultProxySelector = getDefaultProxySelector();
-        if (defaultProxySelector != null) {
-            defaultProxySelector.connectFailed(uri, sa, ioe);
+        if (executeConnectionFailed) {
+            /**
+             * Just try to make the behavior of jre proxy selector same like eclipse proxy selector
+             */
+            ProxySelector defaultProxySelector = getDefaultProxySelector();
+            if (defaultProxySelector != null) {
+                defaultProxySelector.connectFailed(uri, sa, ioe);
+            }
         }
     }
 
-    public static abstract class AbstractProxySelectorProvider implements IProxySelectorProvider {
-
-        @Override
-        public void connectFailed(final URI uri, final SocketAddress sa, final IOException ioe) {
-            // nothing to do
+    public IProxySelectorProvider createDefaultProxySelectorProvider() {
+        if (disableDefaultSelectorProvider) {
+            return null;
         }
+        return new DefaultProxySelectorProvider(Thread.currentThread());
+    }
 
+    public IProxySelectorProvider createDefaultProxySelectorProvider(String host) {
+        if (disableDefaultSelectorProvider) {
+            return null;
+        }
+        return new DefaultProxySelectorProvider(host);
     }
 
     public static interface IProxySelectorProvider {
 
+        Object getKey();
+
         boolean canHandle(final URI uri);
 
         List<Proxy> select(final URI uri);
@@ -176,4 +648,120 @@ public class TalendProxySelector extends ProxySelector {
 
     }
 
+    public static abstract class AbstractProxySelectorProvider implements IProxySelectorProvider {
+
+        private boolean isDebugMode = CommonsPlugin.isDebugMode();
+
+        @Override
+        public Object getKey() {
+            return this;
+        }
+
+        @Override
+        public void connectFailed(final URI uri, final SocketAddress sa, final IOException ioe) {
+            if (isDebugMode) {
+                ExceptionHandler.process(ioe);
+            }
+        }
+
+    }
+
+    public static Collection<Object> getPossibleKeys(URI uri) {
+        Collection<Object> possibleKeys = new ArrayList<>();
+        possibleKeys.add(Thread.currentThread());
+        if (uri != null) {
+            String uriHost = uri.getHost();
+            if (StringUtils.isNotBlank(uriHost)) {
+                possibleKeys.add(uriHost);
+            }
+        }
+        return possibleKeys;
+    }
+
+    private enum EProxySelector {
+        eclipse_default,
+        jre
+    }
+
+    private class DefaultProxySelectorProvider extends TalendProxySelector.AbstractProxySelectorProvider {
+
+        private Thread currentThread = null;
+
+        private String host = null;
+
+        public DefaultProxySelectorProvider(Thread thread) {
+            this.currentThread = thread;
+        }
+
+        public DefaultProxySelectorProvider(String host) {
+            this.host = host;
+            if (StringUtils.isNotBlank(this.host)) {
+                this.host = this.host.toLowerCase();
+            }
+        }
+
+        @Override
+        public Object getKey() {
+            if (this.currentThread != null) {
+                return currentThread;
+            }
+            if (this.host != null) {
+                return this.host;
+            }
+            return super.getKey();
+        }
+
+        @Override
+        public boolean canHandle(URI uri) {
+            if (disableDefaultSelectorProvider) {
+                return false;
+            }
+            if (currentThread != null && Thread.currentThread() == currentThread) {
+                return true;
+            }
+            if (host != null) {
+                if (uri == null) {
+                    return false;
+                }
+                String uriHost = uri.getHost();
+                if (StringUtils.isNotBlank(uriHost)) {
+                    return this.host.equals(uriHost.toLowerCase());
+                }
+                return false;
+            }
+            return false;
+        }
+
+        @Override
+        public List<Proxy> select(URI uri) {
+            List<Proxy> result = new ArrayList<>();
+            try {
+                ProxySelector defaultProxySelector = getDefaultProxySelector();
+                if (defaultProxySelector != null) {
+                    List<Proxy> defaultProxys = defaultProxySelector.select(uri);
+                    if (defaultProxys != null && !defaultProxys.isEmpty()) {
+                        for (Proxy proxy : defaultProxys) {
+                            SocketAddress addr = null;
+                            Proxy.Type proxyType = null;
+                            if (proxy != null) {
+                                proxyType = proxy.type();
+                                addr = proxy.address();
+                            }
+                            if (Proxy.Type.DIRECT == proxyType || (addr != null
+                                    && StringUtils.equals(uri.getHost(), ((InetSocketAddress) addr).getHostString()))) {
+                                result.add(Proxy.NO_PROXY);
+                            } else {
+                                result.add(proxy);
+                            }
+                        }
+                    }
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+            return result;
+        }
+
+    }
+
 }

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/linking/TreeToTablesLinker.java

@@ -33,6 +33,7 @@ import org.eclipse.swt.widgets.TreeItem;
 import org.talend.commons.ui.runtime.i18n.Messages;
 import org.talend.commons.ui.runtime.utils.TableUtils;
 import org.talend.commons.ui.runtime.ws.WindowSystem;
+import org.talend.commons.ui.swt.advanced.dataeditor.AbstractDataTableEditorView;
 import org.talend.commons.ui.swt.drawing.background.IBackgroundRefresher;
 import org.talend.commons.ui.swt.drawing.background.IBgDrawableComposite;
 import org.talend.commons.ui.swt.drawing.link.BezierHorizontalLink;
@@ -464,4 +465,19 @@ public class TreeToTablesLinker<D1, D2> extends BgDrawableComposite implements I
         }
     }
 
+    protected <B> void loadItemDataForLazyLoad(AbstractDataTableEditorView<B> tableEditorView) {
+        if (!tableEditorView.getTableViewerCreator().isLazyLoad()) {
+            return;
+        }
+        List<B> beansList = tableEditorView.getExtendedTableModel().getBeansList();
+        Table table = tableEditorView.getTable();
+        for (TableItem tableItem : table.getItems()) {
+            if (tableItem.getData() == null) {
+                int index = table.indexOf(tableItem);
+                B schemaTarget = beansList.get(index);
+                tableItem.setData(schemaTarget);
+            }
+        }
+    }
+
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/constants/FileConstants.java

@@ -29,6 +29,8 @@ public interface FileConstants {
 
     String RELATIONSHIP_FILE_NAME = "relationship.index"; //$NON-NLS-1$
     
+    String COMPONENT_FILE_NAME = "component.index"; //$NON-NLS-1$
+
     String TALEND_RECYCLE_BIN_INDEX = "recycle_bin.index"; //$NON-NLS-1$
     
     String MIGRATION_TASK_FILE_NAME = "migration_task.index"; //$NON-NLS-1$

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java

@@ -71,6 +71,7 @@ import org.talend.commons.ui.runtime.CommonUIPlugin;
 import org.talend.commons.ui.runtime.exception.MessageBoxExceptionHandler;
 import org.talend.commons.utils.data.container.RootContainer;
 import org.talend.commons.utils.generation.JavaUtils;
+import org.talend.commons.utils.network.TalendProxySelector;
 import org.talend.commons.utils.time.TimeMeasure;
 import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.AbstractDQModelService;
@@ -1896,6 +1897,11 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
 
             TimeMeasure.begin("logOnProject"); //$NON-NLS-1$
             try {
+                /**
+                 * init/check proxy selector, in case default proxy selector is not registed yet
+                 */
+                TalendProxySelector.checkProxy();
+
                 System.getProperties().put("ReadOnlyUser", Boolean.FALSE.toString()); //$NON-NLS-1$
 
                 // remove the auto-build to enhance the build speed and application's use
@@ -2094,7 +2100,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                  */
                 try {
                     ITaCoKitService tacokitService = ITaCoKitService.getInstance();
-                    if (tacokitService != null) {
+                    if (tacokitService != null && tacokitService.isNeedMigration(project.getTechnicalLabel())) {
                         tacokitService.checkMigration(monitor);
                     }
                 } catch (Exception e) {
@@ -2267,9 +2273,14 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
     @Override
     @SuppressWarnings("unchecked")
     public void executeRepositoryWorkUnit(RepositoryWorkUnit workUnit) {
+        checkProxySettings();
         this.repositoryFactoryFromProvider.executeRepositoryWorkUnit(workUnit);
     }
 
+    private void checkProxySettings() {
+        TalendProxySelector.checkProxy();
+    }
+
     @Override
     public void unloadResources(Property property) throws PersistenceException {
         repositoryFactoryFromProvider.unloadResources(property);

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/ComponentsJsonModel.java

@@ -0,0 +1,124 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2018 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.repository.utils;
+
+import java.util.List;
+import java.util.Set;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * DOC jding  class global comment. Detailled comment
+ */
+public class ComponentsJsonModel {
+
+    @JsonProperty("id")
+    private String id;
+
+    @JsonProperty("version")
+    private String version;
+
+    @JsonProperty("parentId")
+    private String parentId;
+
+    @JsonProperty("configurationType")
+    private String configurationType;
+
+    @JsonProperty("name")
+    private String name;
+
+    @JsonProperty("displayName")
+    private String displayName;
+
+    @JsonProperty("edges")
+    private Set edges;
+
+    @JsonProperty("properties")
+    private List properties;
+
+    @JsonProperty("actions")
+    private Object actions;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getVersion() {
+        return version;
+    }
+
+    public void setVersion(String version) {
+        this.version = version;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getDisplayName() {
+        return displayName;
+    }
+
+    public void setDisplayName(String displayName) {
+        this.displayName = displayName;
+    }
+
+    public String getParentId() {
+        return parentId;
+    }
+
+    public void setParentId(String parentId) {
+        this.parentId = parentId;
+    }
+
+    public String getConfigurationType() {
+        return configurationType;
+    }
+
+    public void setConfigurationType(String configurationType) {
+        this.configurationType = configurationType;
+    }
+
+    public Set getEdges() {
+        return edges;
+    }
+
+    public void setEdges(Set edges) {
+        this.edges = edges;
+    }
+
+    public List getProperties() {
+        return properties;
+    }
+
+    public void setProperties(List properties) {
+        this.properties = properties;
+    }
+
+    public Object getActions() {
+        return actions;
+    }
+
+    public void setActions(Object actions) {
+        this.actions = actions;
+    }
+
+}

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/ProjectDataJsonProvider.java

@@ -18,8 +18,14 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
 import org.eclipse.core.resources.IContainer;
@@ -73,6 +79,12 @@ public class ProjectDataJsonProvider {
         return strBuilder.toString();
     }
 
+    public static String getMigrationTaskIndexPath() {
+        StringBuilder strBuilder = new StringBuilder();
+        strBuilder.append(FileConstants.SETTINGS_FOLDER_NAME).append("/").append(FileConstants.MIGRATION_TASK_FILE_NAME); //$NON-NLS-1$
+        return strBuilder.toString();
+    }
+
     public static void saveProjectData(Project project) throws PersistenceException {
         saveProjectData(project, CONTENT_ALL);
     }
@@ -98,6 +110,54 @@ public class ProjectDataJsonProvider {
         }
     }
 
+    public static void saveConfigComponent(String projectLabel, List<ComponentsJsonModel> componentJsons)
+            throws PersistenceException {
+        Collections.sort(componentJsons, new Comparator<ComponentsJsonModel>() {
+
+            @Override
+            public int compare(ComponentsJsonModel configTypeNode1, ComponentsJsonModel configTypeNode2) {
+                return configTypeNode1.getId().compareTo(configTypeNode2.getId());
+            }
+
+        });
+
+        File file = getSavingConfigurationFile(projectLabel, FileConstants.COMPONENT_FILE_NAME);
+        try {
+            if (!file.exists()) {
+                file.createNewFile();
+            }
+            ObjectMapper objectMapper = new ObjectMapper();
+            objectMapper.writerWithDefaultPrettyPrinter().writeValue(file, componentJsons);
+            ResourceUtils.getProject(projectLabel).getFolder(FileConstants.SETTINGS_FOLDER_NAME).refreshLocal(1, null);
+        } catch (Exception e) {
+            throw new PersistenceException(e);
+        }
+    }
+
+    public static Map<String, String[]> getLastConfigComponent(String projectLabel) throws PersistenceException {
+        Map<String, String[]> componentMap = new HashMap<>();
+        File file = getSavingConfigurationFile(projectLabel, FileConstants.COMPONENT_FILE_NAME);
+        TypeReference<List<ComponentsJsonModel>> typeReference = new TypeReference<List<ComponentsJsonModel>>() {
+        };
+        List<ComponentsJsonModel> componentsJsons = null;
+        if (file != null && file.exists()) {
+            try {
+                FileInputStream input = new FileInputStream(file);
+                componentsJsons = new ObjectMapper().readValue(input, typeReference);
+            } catch (IOException e) {
+                throw new PersistenceException(e);
+            }
+            if (componentsJsons != null && componentsJsons.size() > 0) {
+                for (ComponentsJsonModel component : componentsJsons) {
+                    String[] content = new String[] { component.getName(), component.getVersion(), component.getDisplayName() };
+                    componentMap.put(component.getId(), content);
+                }
+            }
+        }
+        
+        return componentMap;
+    }
+
     public static void loadProjectData(Project project, IPath projectFolderPath, int loadContent) throws PersistenceException {
         if ((loadContent & CONTENT_PROJECTSETTING) > 0) {
             loadProjectSettings(project, projectFolderPath);
@@ -221,20 +281,37 @@ public class ProjectDataJsonProvider {
 
     public static void loadMigrationTaskSetting(Project project, InputStream input) throws PersistenceException {
         try {
+        	project.eSetDeliver(false);
             MigrationTaskSetting migrationTaskSetting = null;
             if (input != null) {
                 migrationTaskSetting = new ObjectMapper().readValue(input, MigrationTaskSetting.class);
             }
             if (migrationTaskSetting != null) {
                 MigrationTask fakeTask = createFakeMigrationTask();
-                List<MigrationTask> allRealTask = new ArrayList<MigrationTask>();
+                MigrationTask existingFakeTask = null;
                 for (int i = 0; i < project.getMigrationTask().size(); i++) {
                     MigrationTask task = (MigrationTask) project.getMigrationTask().get(i);
-                    if (!StringUtils.equals(fakeTask.getId(), task.getId())) {
-                        allRealTask.add(task);
+                    if (StringUtils.equals(fakeTask.getId(), task.getId())) {
+                    	existingFakeTask = task;
+                    	break;
                     }
                 }
-                project.getMigrationTask().removeAll(allRealTask);
+                // remove all the migrations from the project
+                project.getMigrationTask().clear();
+                if (existingFakeTask != null) {
+                	// keep only the fake migration task of 7.1.1 with new migration index
+                	// re-use the same instance to make sure we keep the same emf id.
+                	project.getMigrationTask().add(existingFakeTask);
+                }
+                Set<String> tasksIds = new HashSet<>();
+                Iterator<MigrationTaskJson> it = migrationTaskSetting.getMigrationTaskList().iterator();
+                while (it.hasNext()) { // remove duplicates to fix issues found in TUP-22735
+                	MigrationTaskJson task = it.next();
+                	if (tasksIds.contains(task.getId())) {
+                		it.remove();
+                	}
+                	tasksIds.add(task.getId());
+                }
                 project.getMigrationTasks().clear();
                 if (migrationTaskSetting.getMigrationTaskList() != null) {
                     for (MigrationTaskJson json : migrationTaskSetting.getMigrationTaskList()) {
@@ -252,6 +329,7 @@ public class ProjectDataJsonProvider {
         } catch (Exception e) {
             throw new PersistenceException(e);
         } finally {
+            project.eSetDeliver(true);
             closeInputStream(input);
         }
     }
@@ -1105,4 +1183,4 @@ class MigrationTaskJson {
         return task;
     }
 
-}
+}

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/ProjectHelper.java

@@ -131,4 +131,22 @@ public class ProjectHelper {
         }
         return null;
     }
+
+    /**
+     * define the different value for the project types the string value in the licence files Matcht the java enum name
+     * this is a copy of org.talend.commons.model.KeyConstants.ProjectType to avoid adding the all library
+     *
+     */
+    public static enum ProjectType {
+        DI,
+        DQ,
+        MDM;
+    }
+
+    public static int getProjectTypeOrdinal(org.talend.core.model.properties.Project project) {
+        if (project != null && project.getType() != null) {
+            return ProjectType.valueOf(project.getType()).ordinal();
+        }
+        return 0;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ILibraryManagerService.java

@@ -202,6 +202,18 @@ public interface ILibraryManagerService extends IService {
      */
     public boolean isJarNeedToBeDeployed(File jarFile);
 
+    public boolean isSameFile(File f1, File f2);
+
     public void checkModuleStatus(ModuleNeeded module);
 
+    public void guessMavenRUIFromIndex(File jarFile, Map<String, String> sourceAndMavenUri);
+
+    public static ILibraryManagerService get() {
+        GlobalServiceRegister register = GlobalServiceRegister.getDefault();
+        if (register.isServiceRegistered(ILibraryManagerService.class)) {
+            return (ILibraryManagerService) register.getService(ILibraryManagerService.class);
+        }
+        return null;
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/context/RepositoryContext.java

@@ -44,6 +44,8 @@ public class RepositoryContext {
 
     private Map<String, String> fields;
 
+    private boolean token = false;
+
     /**
      * DOC smallet RepositoryContext constructor comment.
      * 
@@ -246,4 +248,12 @@ public class RepositoryContext {
     public void setNoUpdateWhenLogon(boolean noUpdateWhenLogon) {
         this.noUpdateWhenLogon = noUpdateWhenLogon;
     }
+
+    public boolean isToken() {
+        return this.token;
+    }
+
+    public void setToken(boolean token) {
+        this.token = token;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/version/EDatabaseVersion4Drivers.java

@@ -162,9 +162,10 @@ public enum EDatabaseVersion4Drivers {
 
     MAPRDB(new DbVersion4Drivers(EDatabaseTypeName.MAPRDB, new String[] {})),
 
-    REDSHIFT(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT,"redshift","REDSHIFT", "RedshiftJDBC42-no-awssdk-1.2.16.1027.jar")), //$NON-NLS-1$
+    REDSHIFT(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT, "redshift", "REDSHIFT", //$NON-NLS-1$
+            "redshiftJDBC42-no-awssdk-1.2.16.1027.jar")),
     REDSHIFT_SSO(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT_SSO,"redshift sso","REDSHIFT_SSO",
-            new String[] { "RedshiftJDBC42-no-awssdk-1.2.16.1027.jar", "aws-java-sdk-1.11.406.jar", "jackson-core-2.9.5.jar",
+            new String[] { "redshiftJDBC42-no-awssdk-1.2.16.1027.jar", "aws-java-sdk-1.11.406.jar", "jackson-core-2.9.5.jar",
                     "jackson-databind-2.9.5.jar", "jackson-annotations-2.9.0.jar", "httpcore-4.4.9.jar", "httpclient-4.5.5.jar",
                     "joda-time-2.8.1.jar", "commons-logging-1.1.3.jar" })),
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/components/IComponentsFactory.java

@@ -55,6 +55,8 @@ public interface IComponentsFactory {
      */
     public Set<IComponent> getComponents();
 
+    public boolean isInitialising();
+
     /**
      * Get a readonly components collection to avoid ConcurrentModificationException caused by multiple thread
      * access;<br/>

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/general/ConnectionBean.java

@@ -54,6 +54,9 @@ public class ConnectionBean implements Cloneable {
 
     private Map<String, String> dynamicFields = new HashMap<String, String>();
 
+    private static final String TOKEN = "token"; //$NON-NLS-1$
+
+    private static final String URL = "url"; //$NON-NLS-1$
     /**
      * DOC smallet ConnectionBean constructor comment.
      */
@@ -281,6 +284,25 @@ public class ConnectionBean implements Cloneable {
         }
     }
 
+    public boolean isToken() {
+        try {
+            if (conDetails.has(TOKEN)) {
+                return (Boolean) conDetails.get(TOKEN);
+            }
+        } catch (JSONException e) {
+            ExceptionHandler.process(e);
+        }
+        return false;
+    }
+
+    public void setToken(boolean token) {
+        try {
+            conDetails.put(TOKEN, token);
+        } catch (JSONException e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
     @Override
     public ConnectionBean clone() throws CloneNotSupportedException {
         return writeFromJSON(this.getConDetails());
@@ -303,6 +325,7 @@ public class ConnectionBean implements Cloneable {
             toReturn.setPassword(st[i++]);
             toReturn.setWorkSpace(st[i++]);
             toReturn.setComplete(new Boolean(st[i++]));
+            toReturn.setToken(new Boolean(st[i++]));
             JSONObject dynamicJson = new JSONObject();
             toReturn.getConDetails().put(DYNAMICFIELDS, dynamicJson);
             while (i < st.length) {
@@ -374,4 +397,15 @@ public class ConnectionBean implements Cloneable {
         }
     }
 
+    public String getUrl() {
+        try {
+            if (conDetails.has(URL)) {
+                return conDetails.getString(URL);
+            }
+        } catch (JSONException e) {
+            ExceptionHandler.process(e);
+        }
+        return "";
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/general/ModuleNeeded.java

@@ -46,6 +46,8 @@ public class ModuleNeeded {
 
     private boolean required;
 
+    private boolean excluded;
+
     private boolean mrRequired = false; // That indicates if the module is
                                         // required by M/R job.
 
@@ -83,6 +85,13 @@ public class ModuleNeeded {
 
     public static final String UNKNOWN = "Unknown";
 
+    /**
+     * TODO This is a hot fix for 7.2.1 . if the maven url is not specified in component/extension point , we will
+     * generate a release version with this flag=true .Need to improve it after release ,normally the studio should use
+     * all jars with release version by default if maven url is not specified.
+     */
+    private boolean useReleaseVersion = false;
+
     ILibraryManagerService libManagerService = (ILibraryManagerService) GlobalServiceRegister.getDefault()
             .getService(ILibraryManagerService.class);
 
@@ -191,6 +200,7 @@ public class ModuleNeeded {
         cloned.required = required;
         cloned.requiredIf = requiredIf;
         cloned.status = status;
+        cloned.useReleaseVersion = useReleaseVersion;
 
         return cloned;
     }
@@ -293,6 +303,14 @@ public class ModuleNeeded {
         this.required = required;
     }
 
+    public boolean isExcluded() {
+        return this.excluded;
+    }
+
+    public void setExcluded(boolean excluded) {
+        this.excluded = excluded;
+    }
+
     public ELibraryInstallStatus getStatus() {
         ILibraryManagerService libManagerService = (ILibraryManagerService) GlobalServiceRegister.getDefault()
                 .getService(ILibraryManagerService.class);
@@ -424,9 +442,9 @@ public class ModuleNeeded {
         if (this.getModuleLocaion() != null) {
             hashCode *= this.getModuleLocaion().hashCode();
         }
-
-        hashCode *= this.getDefaultMavenURI().hashCode();
-
+        if(this.getDefaultMavenURI() != null){
+            hashCode *= this.getDefaultMavenURI().hashCode();
+        }
         return hashCode;
     }
 
@@ -509,8 +527,16 @@ public class ModuleNeeded {
         }
 
         // maven uri
-        if (!other.getDefaultMavenURI().equals(this.getDefaultMavenURI())) {
-            return false;
+        if (other.getDefaultMavenURI() == null) {
+            if (this.getDefaultMavenURI() != null) {
+                return false;
+            }
+        } else {
+            if (this.getDefaultMavenURI() == null) {
+                return false;
+            } else if (!other.getDefaultMavenURI().equals(this.getDefaultMavenURI())) {
+                return false;
+            }
         }
 
         return true;
@@ -560,7 +586,7 @@ public class ModuleNeeded {
                 if (mvnUrisFromIndex != null) {
                     mavenUri = MavenUrlHelper.addTypeForMavenUri(mvnUrisFromIndex, getModuleName());
                 } else {
-                    mavenUri = MavenUrlHelper.generateMvnUrlForJarName(getModuleName(), true, true);
+                    mavenUri = MavenUrlHelper.generateMvnUrlForJarName(getModuleName(), true, !useReleaseVersion);
                 }
             } else {
                 mavenUri = mavenUriFromConfiguration;
@@ -667,4 +693,8 @@ public class ModuleNeeded {
         this.excludeDependencies = excludeDependencies;
     }
 
+    public void setUseReleaseVersion(boolean useReleaseVersion) {
+        this.useReleaseVersion = useReleaseVersion;
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/ISAPConstant.java

@@ -45,6 +45,14 @@ public interface ISAPConstant {
 
     public static final String CHANGING = "CHANGING";//$NON-NLS-1$
 
+    public static final String PARAMETER_TYPE = "PARAMETER_TYPE";//$NON-NLS-1$
+
+    public static final String PARAMETER_TYPE_IMPORT = "IMPORT";//$NON-NLS-1$
+
+    public static final String PARAMETER_TYPE_CHANGING = "CHANGING";//$NON-NLS-1$
+
+    public static final String PARAMETER_TYPE_TABLES = "TABLES";//$NON-NLS-1$
+
     /*
      * 
      */

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataTable.java

@@ -153,7 +153,7 @@ public class MetadataTable implements IMetadataTable, Cloneable {
             List<IMetadataColumn> temp = new ArrayList<IMetadataColumn>();
             temp.addAll(this.listColumns);
             temp.addAll(this.unusedColumns);
-            if (isRepository && originalColumns != null) {
+            if (originalColumns != null && isRepository) {
                 Collections.sort(temp, new Comparator<IMetadataColumn>() {
 
                     @Override
@@ -213,8 +213,9 @@ public class MetadataTable implements IMetadataTable, Cloneable {
             }
             if (originalColumns != null) {
                 List<String> clonedOriginalColumns = new ArrayList<String>(originalColumns);
-                clonedMetadata.setOriginalColumns(clonedOriginalColumns);
-
+                if(clonedMetadata.getOriginalColumns() == null || clonedMetadata.getOriginalColumns().isEmpty()) {
+                	clonedMetadata.setOriginalColumns(clonedOriginalColumns);
+                }
             }
             clonedMetadata.setTableName(this.getTableName());
             clonedMetadata.setLabel(this.getLabel());

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolAvroHelper.java

@@ -211,6 +211,7 @@ public final class MetadataToolAvroHelper {
             //ignore it now as we can't process the complex expression for the default value, and the default value is not useful for runtime like the old javajet tjdbcxxx
             //TODO support the expression calculate, not sure it's necessary and sometimes, more complex like globalMap.get(xxx) which only have meaning after running the job.
             ExceptionHandler.process(e, Level.WARN);
+            defaultValue = null;
         }
 
         // Types with Document/Unknown elements, store as binary

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolHelper.java

@@ -15,6 +15,7 @@ package org.talend.core.model.metadata;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -614,13 +615,71 @@ public final class MetadataToolHelper {
         target.getListColumns().addAll(columnsTAdd);
         target.sortCustomColumns();
         target.setLabel(source.getLabel());
-        target.setOriginalColumns(source.getOriginalColumns());
+        target.setDbms(source.getDbms());
+        setTargetOriginalColumns(source, target);
+        // List<String> originalColumnsList = null;
+        // if (source.getOriginalColumns() != null) {
+        // originalColumnsList = new ArrayList<String>();
+        // originalColumnsList.addAll(source.getOriginalColumns());
+        // }
+        // target.setOriginalColumns(originalColumnsList);
         Map<String, String> targetProperties = target.getAdditionalProperties();
         Map<String, String> sourceProperties = source.getAdditionalProperties();
         for (Entry<String, String> entry : sourceProperties.entrySet()) {
             targetProperties.put(entry.getKey(), entry.getValue());
         }
     }
+    
+    public static void setTargetOriginalColumns(IMetadataTable source, IMetadataTable target) {
+    	List<String> sColumns = source.getOriginalColumns();
+    	List<String> tColumns = target.getOriginalColumns();
+    	if(sColumns == null) {
+    		return;
+    	}
+    	if(tColumns == null) {
+    		target.setOriginalColumns(sColumns);
+    		return;
+    	}
+    	
+    	if(sColumns.size() == tColumns.size()) {
+    		boolean same = true;
+    		for(int i = 0;i<sColumns.size();i++) {
+    			if(!sColumns.get(i).equals(tColumns.get(i))) {
+    				same = false;
+    				break;
+    			}
+    		}
+    		if(same) {
+    			return;
+    		}
+    	}
+    	
+    	for(String sColumn : sColumns) {
+    		if(tColumns.contains(sColumn)) {
+    			continue;
+    		}
+    		tColumns.add(sColumn);
+    	}
+    	
+    	List<IMetadataColumn> targetColumns = target.getListColumns();
+    	List<String> temp = new ArrayList<String>(tColumns);
+    	if (targetColumns != null) {
+    		List<String> columnNames = new ArrayList<String>();
+        	for(IMetadataColumn column : targetColumns){
+        		columnNames.add(column.getLabel());
+        	}
+            Collections.sort(temp, new Comparator<String>() {
+
+                @Override
+                public int compare(String o1, String o2) {
+                    int index1 = columnNames.indexOf(o1);
+                    int index2 = columnNames.indexOf(o2);
+                    return index1 - index2;
+                }
+            });
+        }
+    	target.setOriginalColumns(temp);
+    }
 
     public static void copyTable(List<IMetadataColumn> sourceColumns, IMetadataTable target, List<IMetadataColumn> targetColumns) {
         if (sourceColumns == null || target == null || targetColumns == null) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/migration/AbstractItemMigrationTask.java

@@ -146,7 +146,7 @@ public abstract class AbstractItemMigrationTask extends AbstractMigrationTask im
      * this returns all type handled by this migration task even the extended type that could benefit from this
      * migrations
      * */
-    List<ERepositoryObjectType> getAllTypes() {
+    protected List<ERepositoryObjectType> getAllTypes() {
         List<ERepositoryObjectType> declaredTypes = getTypes();
         ArrayList<ERepositoryObjectType> allTypes = new ArrayList<ERepositoryObjectType>(declaredTypes.size());
         allTypes.addAll(declaredTypes);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/AbstractNode.java

@@ -308,7 +308,8 @@ public abstract class AbstractNode implements INode {
                         return mapMerge.keySet().iterator().next().getSubProcessStartNode(withConditions);
                     }
                 }
-                if ((getCurrentActiveLinksNbInput(EConnectionType.MAIN) == 0)) {
+                if ((getCurrentActiveLinksNbInput(EConnectionType.MAIN) == 0)
+                        && !checkIfCurrentActiveLinksIsLookup()) {
                     return this; // main branch here, so we got the correct sub
                                  // process start.
                 }
@@ -327,9 +328,13 @@ public abstract class AbstractNode implements INode {
 
         for (IConnection connec : getIncomingConnections()) {
             if (((AbstractNode) connec.getSource()).isOnMainMergeBranch()) {
+                // with lookup, tMap incoming connection contains FLOW_MAIN and FLOW_REF,
+                // should take care of FLOW_MAIN only
                 if (!connec.getLineStyle().equals(EConnectionType.FLOW_REF)) {
                     return connec.getSource().getSubProcessStartNode(withConditions);
                 }
+            } else if (connec.getLineStyle().equals(EConnectionType.RUN_AFTER)) {
+            	return connec.getSource().getSubProcessStartNode(withConditions);
             }
         }
         return null;
@@ -345,6 +350,27 @@ public abstract class AbstractNode implements INode {
         return nb;
     }
 
+    private boolean checkIfCurrentActiveLinksIsLookup() {
+        boolean flag = false;
+        int input = 0;
+        for (IConnection inConnection : getIncomingConnections()) {
+            // refer to DataProcess.checkFlowRefLink() added RUN_AFTER for incoming connection
+            if (inConnection.isActivate() && inConnection.getLineStyle().getId() == EConnectionType.RUN_AFTER.getId()) {
+                input++;
+            }
+        }
+        // check if run_after is for lookup
+        if (input > 0) {
+            for (IConnection outConnection : getOutgoingConnections()) {
+                if (outConnection.isActivate() && outConnection.getLineStyle().getId() == EConnectionType.FLOW_REF.getId()) {
+                    flag = true;
+                    break;
+                }
+            }
+        }
+        return flag;
+    }
+
     /*
      * (non-Javadoc)
      * 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ProcessUtils.java

@@ -20,11 +20,11 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 
-import org.eclipse.emf.common.util.EList;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.utils.VersionUtils;
 import org.talend.core.GlobalServiceRegister;
+import org.talend.core.IESBService;
 import org.talend.core.ITDQItemService;
 import org.talend.core.PluginChecker;
 import org.talend.core.hadoop.IHadoopClusterService;
@@ -33,7 +33,6 @@ import org.talend.core.model.general.Project;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.ProcessItem;
-import org.talend.core.model.properties.ProjectReference;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.properties.SQLPatternItem;
 import org.talend.core.model.relationship.Relation;
@@ -53,7 +52,6 @@ import org.talend.core.utils.BitwiseOptionUtils;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextType;
-import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.MetadataType;
 import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
 import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
@@ -897,33 +895,14 @@ public final class ProcessUtils {
                     ERepositoryObjectType itemType = ERepositoryObjectType.getItemType(property.getItem());
                     // route job
                     if (itemType != null && (itemType.equals(ERepositoryObjectType.PROCESS_ROUTE)
-                    		|| itemType.equals(ERepositoryObjectType.PROCESS_ROUTELET))) {
+                            || itemType.equals(ERepositoryObjectType.PROCESS_ROUTELET)
+                            || "CAMEL".equals(process.getComponentsType()))) {
                         needBeans = true;
                     }
                 }
             }
         }
-
-        if (needBeans && GlobalServiceRegister.getDefault().isServiceRegistered(IProxyRepositoryService.class)) {
-            IProxyRepositoryService service = (IProxyRepositoryService) GlobalServiceRegister.getDefault()
-                    .getService(IProxyRepositoryService.class);
-            ERepositoryObjectType beansType = ERepositoryObjectType.valueOf("BEANS"); //$NON-NLS-1$
-            try {
-                IProxyRepositoryFactory factory = service.getProxyRepositoryFactory();
-                List<IRepositoryViewObject> all = factory.getAll(project, beansType);
-                List<ProjectReference> references = ProjectManager.getInstance().getCurrentProject()
-                        .getProjectReferenceList(true);
-                for (ProjectReference ref : references) {
-                    all.addAll(factory.getAll(new Project(ref.getReferencedProject()), beansType));
-                }
-                if (!all.isEmpty()) { // has bean
-                    return true;
-                }
-            } catch (PersistenceException e) {
-                ExceptionHandler.process(e);
-            }
-        }
-        return false;
+        return needBeans && GlobalServiceRegister.getDefault().isServiceRegistered(IESBService.class);
     }
 
     public static boolean isRequiredPigUDFs(IProcess process) {
@@ -974,5 +953,17 @@ public final class ProcessUtils {
 
         return false;
     }
+    
+    public static boolean isChildRouteProcess(IProcess process) {
+        if (process != null)   {
+            for (INode node : process.getGraphicalNodes()) {
+                if ((node.getComponent().getName() != null) && 
+                        (node.getComponent().getName().compareTo("tRouteInput") == 0)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
 
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/relationship/RelationshipItemBuilder.java

@@ -23,6 +23,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Level;
@@ -128,8 +129,6 @@ public class RelationshipItemBuilder {
 
     private Map<Relation, Set<Relation>> referencesItemsRelations;
 
-    private Map<String, String> hadoopItemReferences = new HashMap<String, String>();
-
     private boolean loaded = false;
 
     private boolean loading = false;
@@ -643,8 +642,8 @@ public class RelationshipItemBuilder {
             return;
         }
         loading = true;
-        currentProjectItemsRelations = new HashMap<Relation, Set<Relation>>();
-        referencesItemsRelations = new HashMap<Relation, Set<Relation>>();
+        currentProjectItemsRelations = new ConcurrentHashMap<Relation, Set<Relation>>();
+        referencesItemsRelations = new ConcurrentHashMap<Relation, Set<Relation>>();
 
         loadRelations(currentProjectItemsRelations, getAimProject());
 
@@ -1097,10 +1096,11 @@ public class RelationshipItemBuilder {
             Map<Relation, Set<Relation>> relations = handler.find(item);
             mergeRelationship(itemRelations, relations);
         }
-
-        if (oldProjectRelations != null) {
-            // check if there is any changes on the relations.
-            Set<Relation> newProjectRelations = currentProjectItemsRelations.get(relation);
+        // check if there is any changes on the relations.
+        Set<Relation> newProjectRelations = currentProjectItemsRelations.get(relation);
+        if (oldProjectRelations == null && newProjectRelations == null) {
+            relationsModified = false;
+        } else if (oldProjectRelations != null) {
             if (oldProjectRelations.size() == newProjectRelations.size()) {
                 relationsModified = false;
                 for (Relation newRelation : newProjectRelations) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/repository/ERepositoryObjectType.java

@@ -427,6 +427,11 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
      */
     public final static ERepositoryObjectType JOBLET = ERepositoryObjectType.valueOf("JOBLET"); //$NON-NLS-1$
 
+    /**
+     * <font color="red">This value may be <b>null</b> in some products, <b>should add NPE check</b></font>
+     */
+    public final static ERepositoryObjectType SERVICES = ERepositoryObjectType.valueOf("SERVICES"); //$NON-NLS-1$
+
     public final static ERepositoryObjectType JOBLET_DESIGNS = ERepositoryObjectType.valueOf("JOBLET_DESIGNS"); //$NON-NLS-1$
 
     public final static ERepositoryObjectType SPARK_JOBLET = ERepositoryObjectType.valueOf("SPARK_JOBLET"); //$NON-NLS-1$
@@ -439,6 +444,8 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public final static ERepositoryObjectType ROUTINES = ERepositoryObjectType.valueOf("ROUTINES"); //$NON-NLS-1$
 
+    public final static ERepositoryObjectType BEANS = ERepositoryObjectType.valueOf("BEANS"); //$NON-NLS-1$
+
     public final static ERepositoryObjectType METADATA_HEADER_FOOTER = ERepositoryObjectType.valueOf("METADATA_HEADER_FOOTER"); //$NON-NLS-1$
 
     public final static ERepositoryObjectType JOB_SCRIPT = ERepositoryObjectType.valueOf("JOB_SCRIPT"); //$NON-NLS-1$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/routines/RoutineLibraryMananger.java

@@ -12,6 +12,9 @@
 // ============================================================================
 package org.talend.core.model.routines;
 
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Enumeration;
@@ -29,9 +32,12 @@ import org.osgi.framework.Bundle;
 import org.talend.commons.utils.resource.FileExtensions;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.ILibraryManagerService;
-import org.talend.core.ISVNProviderServiceInCoreRuntime;
-import org.talend.core.PluginChecker;
 import org.talend.core.model.general.LibraryInfo;
+import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.nexus.TalendLibsServerManager;
+import org.talend.core.runtime.maven.MavenArtifact;
+import org.talend.core.runtime.maven.MavenConstants;
+import org.talend.core.runtime.maven.MavenUrlHelper;
 
 /**
  * wchen class global comment. Detailled comment
@@ -93,7 +99,14 @@ public class RoutineLibraryMananger {
                             if (entry != null && libManagerService != null) {
                                 try {
                                     URL fileUrl = FileLocator.toFileURL(entry);
-                                    libManagerService.deploy(fileUrl.toURI());
+                                    if(fileUrl != null){
+                                        if (!"file".equals(fileUrl.getProtocol())) throw new IllegalArgumentException();
+                                        File file = new File(fileUrl.getFile());
+                                        if (needDeploy(fileUrl)) {
+                                            URI fileUri = file.toURI();
+                                            libManagerService.deploy(fileUri);
+                                        }
+                                    }
                                 } catch (Exception e) {
                                     log.warn("Cannot deploy: " + bundleName + path);
                                     continue;
@@ -108,6 +121,56 @@ public class RoutineLibraryMananger {
         }
     }
 
+    private boolean needDeploy(URL fileUrl) throws IOException, Exception {
+        File file = new File(fileUrl.getFile());
+        ILibraryManagerService libManagerService = null;
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibraryManagerService.class)) {
+            libManagerService = (ILibraryManagerService) GlobalServiceRegister.getDefault()
+                    .getService(ILibraryManagerService.class);
+        }
+        if (libManagerService != null) {
+            Map<String, String> sourceAndMavenUri = new HashMap<>();
+            libManagerService.guessMavenRUIFromIndex(file, sourceAndMavenUri);
+            String mavUri = null;
+            boolean isSnapshot = false;
+            for (String key : sourceAndMavenUri.keySet()) {
+                if (sourceAndMavenUri.get(key).equals(file.getPath())) {
+                    mavUri = key;
+                    break;
+                }
+            }
+            if (mavUri == null) {
+                return true;
+            }
+            final MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(mavUri);
+            if (parseMvnUrl != null) {
+                if (parseMvnUrl.getVersion() != null && parseMvnUrl.getVersion().endsWith(MavenConstants.SNAPSHOT)) {
+                    isSnapshot = true;
+                }
+            }
+            TalendLibsServerManager manager = TalendLibsServerManager.getInstance();
+            ArtifactRepositoryBean customNexusServer = manager.getCustomNexusServer();
+            File jarFile = null;
+            try {
+                jarFile = libManagerService.resolveJar(customNexusServer, mavUri);
+            } catch (Exception ex) {
+                // Ignore here
+            }
+            boolean exist = (jarFile != null && jarFile.exists());
+            if (exist) {
+                if (isSnapshot) {
+                    boolean isSame = libManagerService.isSameFile(jarFile, file);
+                    if (!isSame) {
+                        return true;
+                    }
+                }
+            } else {
+                return true;
+            }
+        }
+        return false;
+    }
+
     public Map<String, List<LibraryInfo>> getRoutineAndJars() {
         if (routineAndJars == null) {
             routineAndJars = new HashMap<String, List<LibraryInfo>>();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/NodeUtil.java

@@ -681,16 +681,15 @@ public class NodeUtil {
             }
             List<? extends IConnection> listInConns = node.getIncomingConnections();
             if (listInConns != null && listInConns.size() > 0) {
-                String retResult = getPrivateConnClassName(listInConns.get(0));
-                if (retResult == null) {
-                    return conn.getName();
-                } else {
-                    return retResult;
+                for (IConnection connection : listInConns) {
+                    if (EConnectionType.FLOW_REF != connection.getLineStyle()) {
+                        String retResult = getPrivateConnClassName(connection);
+                        return retResult != null ? retResult : conn.getName();
+                    }
                 }
             }
         }
         return null;
-
     }
 
     /**

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/ArtifactRepositoryBean.java

@@ -79,6 +79,11 @@ public class ArtifactRepositoryBean implements Cloneable {
                 if (index > 0) {
                     nexusUrl = url.substring(0, index + ARTIFACT_MIDDLE_PATH.length());
                     repoId = StringUtilities.removeEndingString(url.substring(index + ARTIFACT_MIDDLE_PATH.length()), "/");
+                } else {
+                    // can be non-default contextpath or root
+                    String tempurl = StringUtilities.removeEndingString(url, "/");
+                    repoId = tempurl.substring(tempurl.lastIndexOf("/") + 1);
+                    nexusUrl = url.substring(0, url.indexOf(repoId));
                 }
             }
             return new String[] { nexusUrl, repoId };

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/HttpClientTransport.java

@@ -12,14 +12,12 @@
 // ============================================================================
 package org.talend.core.nexus;
 
-import java.net.Authenticator;
 import java.net.InetSocketAddress;
 import java.net.PasswordAuthentication;
 import java.net.Proxy;
 import java.net.Proxy.Type;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
@@ -37,7 +35,6 @@ import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.OperationCanceledException;
 import org.talend.commons.exception.BusinessException;
-import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.utils.network.TalendProxySelector;
 import org.talend.commons.utils.network.TalendProxySelector.IProxySelectorProvider;
 import org.talend.core.runtime.maven.MavenArtifact;
@@ -123,7 +120,7 @@ public abstract class HttpClientTransport {
         }
     }
 
-    private IProxySelectorProvider addProxy(final DefaultHttpClient httpClient, URI requestURI) {
+    public static IProxySelectorProvider addProxy(final DefaultHttpClient httpClient, URI requestURI) {
         IProxySelectorProvider proxySelectorProvider = null;
         try {
             if (Boolean.valueOf(System.getProperty(PROP_PROXY_HTTP_CLIENT_USE_DEFAULT_SETTINGS, Boolean.FALSE.toString()))) {
@@ -141,10 +138,10 @@ public abstract class HttpClientTransport {
                 }
                 final Proxy finalProxy = usedProxy;
                 InetSocketAddress address = (InetSocketAddress) finalProxy.address();
-                String proxyServer = address.getHostName();
+                String proxyServer = address.getHostString();
                 int proxyPort = address.getPort();
-                PasswordAuthentication proxyAuthentication = Authenticator.requestPasswordAuthentication(proxyServer,
-                        address.getAddress(), proxyPort, "Http Proxy", "Http proxy authentication", null);
+                TalendProxySelector proxySelector = TalendProxySelector.getInstance();
+                PasswordAuthentication proxyAuthentication = proxySelector.getHttpPasswordAuthentication();
                 if (proxyAuthentication != null) {
                     String proxyUser = proxyAuthentication.getUserName();
                     String proxyPassword = "";
@@ -157,7 +154,7 @@ public abstract class HttpClientTransport {
                 }
                 HttpHost proxyHost = new HttpHost(proxyServer, proxyPort);
                 httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost);
-                proxySelectorProvider = createProxySelectorProvider();
+                proxySelectorProvider = proxySelector.createDefaultProxySelectorProvider();
             }
             return proxySelectorProvider;
         } finally {
@@ -167,61 +164,12 @@ public abstract class HttpClientTransport {
         }
     }
 
-    private void removeProxy(IProxySelectorProvider proxySelectorProvider) {
+    public static void removeProxy(IProxySelectorProvider proxySelectorProvider) {
         if (proxySelectorProvider != null) {
             TalendProxySelector.getInstance().removeProxySelectorProvider(proxySelectorProvider);
         }
     }
 
-    private IProxySelectorProvider createProxySelectorProvider() {
-        IProxySelectorProvider proxySelectorProvider = new TalendProxySelector.AbstractProxySelectorProvider() {
-
-            private Thread currentThread = Thread.currentThread();
-
-            @Override
-            public List<Proxy> select(URI uri) {
-                // return Collections.EMPTY_LIST;
-
-                List<Proxy> newProxys = new ArrayList<>();
-                if (uri == null) {
-                    return newProxys;
-                }
-                String schema = uri.getScheme();
-                if (schema != null && schema.toLowerCase().startsWith("socket")) { //$NON-NLS-1$
-                    try {
-                        URI newUri = new URI("https", uri.getUserInfo(), uri.getHost(), uri.getPort(), uri.getPath(),
-                                uri.getQuery(), uri.getFragment());
-                        List<Proxy> proxys = TalendProxySelector.getInstance().getDefaultProxySelector().select(newUri);
-                        if (proxys != null && !proxys.isEmpty()) {
-                            newProxys.addAll(proxys);
-                        } else {
-                            newUri = new URI("http", uri.getUserInfo(), uri.getHost(), uri.getPort(), uri.getPath(),
-                                    uri.getQuery(), uri.getFragment());
-                            proxys = TalendProxySelector.getInstance().getDefaultProxySelector().select(newUri);
-                            if (proxys != null && !proxys.isEmpty()) {
-                                newProxys.addAll(proxys);
-                            }
-                        }
-                    } catch (URISyntaxException e) {
-                        ExceptionHandler.process(e);
-                    }
-                }
-                return newProxys;
-
-            }
-
-            @Override
-            public boolean canHandle(URI uri) {
-                if (Thread.currentThread() == currentThread) {
-                    return true;
-                }
-                return false;
-            }
-
-        };
-        return proxySelectorProvider;
-    }
-
     public void processResponseCode(HttpResponse response) throws BusinessException {
         StatusLine statusLine = response.getStatusLine();
         int responseCode = statusLine.getStatusCode();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/IRepositoryArtifactHandler.java

@@ -61,5 +61,7 @@ public interface IRepositoryArtifactHandler {
     public IRepositoryArtifactHandler clone();
 
     public String getRepositoryURL(boolean isRelease);
+    
+    public String resolveRemoteSha1(MavenArtifact artifact, boolean fromRelease) throws Exception;
 
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/NexusConstants.java

@@ -26,5 +26,7 @@ public class NexusConstants {
 
     public static final String SLASH = "/";//$NON-NLS-1$ 
 
-    public static final String SNAPSHOTS = "@snapshots";
+    public static final String SNAPSHOTS = "@snapshots";//$NON-NLS-1$
+
+    public static final String DISALLOW_RELEASES = "@noreleases";//$NON-NLS-1$
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/TalendMavenResolver.java

@@ -23,6 +23,9 @@ import org.osgi.framework.ServiceReference;
 import org.osgi.service.cm.ManagedService;
 import org.osgi.util.tracker.ServiceTracker;
 import org.osgi.util.tracker.ServiceTrackerCustomizer;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.TalendProxySelector;
+import org.talend.commons.utils.network.TalendProxySelector.IProxySelectorProvider;
 import org.talend.core.runtime.CoreRuntimePlugin;
 
 /**
@@ -98,7 +101,24 @@ public class TalendMavenResolver {
     }
 
     public static File resolve(String mvnUri) throws IOException, RuntimeException {
-        return getMavenResolver().resolve(mvnUri);
+        TalendProxySelector selectorInstance = null;
+        IProxySelectorProvider proxySelector = null;
+        try {
+            try {
+                selectorInstance = TalendProxySelector.getInstance();
+                proxySelector = selectorInstance.createDefaultProxySelectorProvider();
+                if (proxySelector != null) {
+                    selectorInstance.addProxySelectorProvider(proxySelector);
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+            return getMavenResolver().resolve(mvnUri);
+        } finally {
+            if (proxySelector != null && selectorInstance != null) {
+                selectorInstance.removeProxySelectorProvider(proxySelector);
+            }
+        }
     }
 
     public static void upload(String groupId, String artifactId, String classifier, String extension, String version,

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/prefs/ITalendCorePrefConstants.java

@@ -192,6 +192,10 @@ public interface ITalendCorePrefConstants {
 
     public static final String NEXUS_REFRESH_FREQUENCY = "NEXUS_REFRESH_FREQUENCY"; //$NON-NLS-1$
 
+    public static final String NEXUS_SHARE_LIBS = "NEXUS_SHARE_LIBS";
+
+    public static final boolean NEXUS_SHARE_LIBS_DEFAULT = true;
+
     /**
      * AutoMap algorithm weight
      */

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenArtifact.java

@@ -24,7 +24,7 @@ public class MavenArtifact implements Cloneable {
     private static final char ARTIFACT_SEPARATOR = '-';
 
     private String repositoryUrl, groupId, artifactId, version, type, classifier, description, url, license, licenseUrl,
-            distribution, username, password, lastUpdated;
+            distribution, username, password, lastUpdated, sha1, md5;
 
     public String getLastUpdated() {
         return this.lastUpdated;
@@ -138,6 +138,22 @@ public class MavenArtifact implements Cloneable {
         this.password = password;
     }
 
+    public String getSha1() {
+        return sha1;
+    }
+
+    public void setSha1(String sha1) {
+        this.sha1 = sha1;
+    }
+
+    public String getMd5() {
+        return md5;
+    }
+
+    public void setMd5(String md5) {
+        this.md5 = md5;
+    }
+
     /**
      * 
      * DOC ggu Comment method "getFileName".

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenConstants.java

@@ -42,6 +42,8 @@ public interface MavenConstants {
     
     static final String POM_FILTER = "POM_FILTER";
 
+    static final String USE_PROFILE_MODULE = "USE_PROFILE_MODULE";
+
     /*
      * for lib
      */
@@ -67,4 +69,5 @@ public interface MavenConstants {
 
     static final String PACKAGING_POM = "pom";
 
+    static final String SKIP_FOLDERS = "SKIP_FOLDERS";
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/process/LastGenerationInfo.java

@@ -352,6 +352,15 @@ public class LastGenerationInfo {
         }
     }
 
+    /**
+     * Clear modules per job cache, not thread safe
+     */
+    public void clearModulesNeededPerJob() {
+        if (!modulesNeededPerJob.isEmpty()) {
+            modulesNeededPerJob.clear();
+        }
+    }
+
     public void clean() {
         modulesNeededPerJob.clear();
         routinesNeededPerJob.clear();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/process/TalendProcessArgumentConstant.java

@@ -62,6 +62,8 @@ public interface TalendProcessArgumentConstant {
 
     static final String ARG_LAUNCHER_NAME = "LAUNCHER_NAME";
 
+    static final String ARG_AVOID_BRANCH_NAME = "AVOID_BRANCH_NAME";
+
     static final String CMD_ARG_STATS_PORT_PARAM = "stat_port";
 
     static final String CMD_ARG_TRACE_PORT_PARAM = "trace_port";

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/process/TalendProcessOptionConstants.java

@@ -69,4 +69,6 @@ public interface TalendProcessOptionConstants {
 
     public static final int MODULES_FOR_MR = 1 << 4;
 
+    public static final int MODULES_EXCLUDE_SHADED = 1 << 5;
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/services/IGenericService.java

@@ -17,6 +17,7 @@ import java.util.Map;
 
 import org.talend.core.IService;
 import org.talend.core.model.process.IElementParameter;
+import org.talend.core.model.process.INode;
 
 /**
  * created by ycbai on 2016年3月24日 Detailled comment
@@ -38,5 +39,7 @@ public interface IGenericService extends IService {
      * 
      */
     public List<Map<String, String>> getAllGenericComponentsInfo();
+    
+    public void resetReferenceValue(INode curNode, String oldConnectionName, String newConnectionName);
 
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/AbstractRoutineSynchronizer.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2018 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
@@ -19,7 +19,9 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
@@ -79,6 +81,8 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
     }
 
     private Collection<RoutineItem> getAll(ERepositoryObjectType type, boolean syncRef) throws SystemException {
+        // init code project
+        getRunProcessService().getTalendCodeJavaProject(type);
         // remove routine with same name in reference project
         final Map<String, RoutineItem> beansList = new HashMap<String, RoutineItem>();
         for (IRepositoryViewObject obj : getRepositoryService().getProxyRepositoryFactory().getAll(type)) {
@@ -95,27 +99,37 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
         return (IRepositoryService) GlobalServiceRegister.getDefault().getService(IRepositoryService.class);
     }
 
-    private void getReferencedProjectRoutine(final Map<String, RoutineItem> beansList, final Project project,
-            ERepositoryObjectType routineType, boolean syncRef) throws SystemException {
-        for (IRepositoryViewObject obj : getRepositoryService().getProxyRepositoryFactory().getAll(project, routineType)) {
+    private Set<IRepositoryViewObject> getReferencedProjectRoutine(final Map<String, RoutineItem> beansList,
+            final Project project, ERepositoryObjectType routineType, boolean syncRef) throws SystemException {
+        // init ref code project
+        if (syncRef) {
+            getRunProcessService().getTalendCodeJavaProject(routineType, project.getTechnicalLabel());
+        }
+        Set<IRepositoryViewObject> routines = new HashSet<>();
+        routines.addAll(getRepositoryService().getProxyRepositoryFactory().getAll(project, routineType));
+        for (IRepositoryViewObject obj : routines) {
             final String key = obj.getProperty().getLabel();
             // it does not have a routine with same name
             if (!beansList.containsKey(key)) {
                 beansList.put(key, (RoutineItem) obj.getProperty().getItem());
             }
-            if (syncRef) {
-                // sync routine
-                syncRoutine((RoutineItem) obj.getProperty().getItem(), false, true, true);
-            }
+        }
+        for (ProjectReference projectReference : project.getProjectReferenceList()) {
+            routines.addAll(getReferencedProjectRoutine(beansList, new Project(projectReference.getReferencedProject()),
+                    routineType, syncRef));
         }
         if (syncRef) {
+            routines.stream().forEach(obj -> {
+                try {
+                    syncRoutine((RoutineItem) obj.getProperty().getItem(), project.getTechnicalLabel(), true, true);
+                } catch (SystemException e) {
+                    ExceptionHandler.process(e);
+                }
+            });
             // sync system routine
             syncSystemRoutine(project);
         }
-
-        for (ProjectReference projectReference : project.getProjectReferenceList()) {
-            getReferencedProjectRoutine(beansList, new Project(projectReference.getReferencedProject()), routineType, syncRef);
-        }
+        return routines;
     }
 
     protected void syncSystemRoutine(Project project) throws SystemException {
@@ -128,24 +142,17 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
     }
 
     protected IFile getRoutineFile(RoutineItem routineItem) throws SystemException {
-        return getRoutineFile(routineItem, true);
+        return getRoutineFile(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel());
     }
 
-    protected IFile getRoutineFile(RoutineItem routineItem, boolean currentProject) throws SystemException {
-        String projectTechName;
-        if (currentProject) {
-            projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
-        } else {
-            projectTechName = ProjectManager.getInstance().getProject(routineItem).getTechnicalLabel();
-        }
+    protected IFile getRoutineFile(RoutineItem routineItem, String projectTechName) throws SystemException {
         ITalendProcessJavaProject talendProcessJavaProject = getRunProcessService()
                 .getTalendCodeJavaProject(ERepositoryObjectType.getItemType(routineItem), projectTechName);
         if (talendProcessJavaProject == null) {
             return null;
         }
         IFolder routineFolder = talendProcessJavaProject.getSrcSubFolder(null, routineItem.getPackageType());
-        IFile file = routineFolder.getFile(routineItem.getProperty().getLabel() + JavaUtils.JAVA_EXTENSION);
-        return file;
+        return routineFolder.getFile(routineItem.getProperty().getLabel() + JavaUtils.JAVA_EXTENSION);
     }
 
     private IFile getProcessFile(ProcessItem item) throws SystemException {
@@ -193,30 +200,31 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
 
     @Override
     public void syncRoutine(RoutineItem routineItem, boolean copyToTemp) throws SystemException {
-        syncRoutine(routineItem, true, copyToTemp, false);
+        syncRoutine(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), copyToTemp, false);
     }
 
     @Override
     public void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceUpdate) throws SystemException {
-        syncRoutine(routineItem, true, copyToTemp, forceUpdate);
+        syncRoutine(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), copyToTemp, forceUpdate);
     }
 
     @Override
-    public void syncRoutine(RoutineItem routineItem, boolean currentProject, boolean copyToTemp, boolean forceUpdate) throws SystemException {
+    public void syncRoutine(RoutineItem routineItem, String projectTechName, boolean copyToTemp, boolean forceUpdate)
+            throws SystemException {
         boolean needSync = false;
         if (routineItem != null) {
             if (forceUpdate || !isRoutineUptodate(routineItem)) {
                 needSync = true;
             } else {
-                IFile file = getRoutineFile(routineItem, currentProject);
+                IFile file = getRoutineFile(routineItem, projectTechName);
                 if (file != null && !file.exists()) {
                     needSync = true;
                 }
             }
         }
         if (needSync) {
-            doSyncRoutine(routineItem, currentProject, copyToTemp);
-            if (currentProject) {
+            doSyncRoutine(routineItem, projectTechName, copyToTemp);
+            if (ProjectManager.getInstance().getCurrentProject().getTechnicalLabel().equals(projectTechName)) {
                 setRoutineAsUptodate(routineItem);
             }
         }
@@ -224,14 +232,14 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
 
     public void syncRoutine(RoutineItem routineItem) throws SystemException {
         if (routineItem != null) {
-            doSyncRoutine(routineItem, true, true);
+            doSyncRoutine(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), true);
             setRoutineAsUptodate(routineItem);
         }
     }
 
-    private void doSyncRoutine(RoutineItem routineItem, boolean currentProject, boolean copyToTemp) throws SystemException {
+    private void doSyncRoutine(RoutineItem routineItem, String projectTechName, boolean copyToTemp) throws SystemException {
         try {
-            IFile file = getRoutineFile(routineItem, currentProject);
+            IFile file = getRoutineFile(routineItem, projectTechName);
             if (file == null) {
                 return;
             }
@@ -358,7 +366,7 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
 
     /*
      * (non-Javadoc)
-     * 
+     *
      * @see org.talend.designer.codegen.ITalendSynchronizer#syncAllPigudf()
      */
     @Override
@@ -374,7 +382,7 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
     @Override
     public void syncAllBeansForLogOn() throws SystemException {
         for (RoutineItem beanItem : getBeans(true)) {
-            syncRoutine(beanItem, true, true, true);
+            syncRoutine(beanItem, true, true);
         }
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/ITalendSynchronizer.java

@@ -45,7 +45,8 @@ public interface ITalendSynchronizer {
 
     void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceUpdate) throws SystemException;
 
-    void syncRoutine(RoutineItem routineItem, boolean currentProject, boolean copyToTemp, boolean forceUpdate) throws SystemException;
+    void syncRoutine(RoutineItem routineItem, String projectTechName, boolean copyToTemp, boolean forceUpdate)
+            throws SystemException;
 
     IFile getFile(Item item) throws SystemException;
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/core/ICamelDesignerCoreService.java

@@ -12,6 +12,8 @@
 // ============================================================================
 package org.talend.designer.core;
 
+import java.util.Collection;
+
 import org.dom4j.Element;
 import org.talend.core.IService;
 import org.talend.core.model.process.EConnectionType;
@@ -60,4 +62,12 @@ public interface ICamelDesignerCoreService extends IService {
 
     public FileItem newRouteDocumentationItem();
 
+    /**
+     * DOC sunchaoqun Comment method "getUnselectDependenciesBundle".
+     * 
+     * @param processItem
+     * @return
+     */
+    Collection<String> getUnselectDependenciesBundle(ProcessItem processItem);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java

@@ -25,6 +25,7 @@ import org.eclipse.jdt.core.IJavaProject;
 import org.eclipse.jface.action.IAction;
 import org.eclipse.jface.preference.IPreferenceStore;
 import org.eclipse.jface.viewers.IStructuredSelection;
+import org.talend.core.GlobalServiceRegister;
 import org.talend.core.IService;
 import org.talend.core.language.ECodeLanguage;
 import org.talend.core.language.ICodeProblemsChecker;
@@ -216,6 +217,8 @@ public interface IRunProcessService extends IService {
 
     ITalendProcessJavaProject getTalendJobJavaProject(Property property);
 
+    IFolder getCodeSrcFolder(ERepositoryObjectType type, String projectTechName);
+
     ITalendProcessJavaProject getTempJavaProject();
 
     void clearProjectRelatedSettings();
@@ -235,4 +238,11 @@ public interface IRunProcessService extends IService {
     public void handleJobDependencyLoop(JobInfo mainJobInfo, List<JobInfo> listJobs, IProgressMonitor progressMonitor)
             throws Exception;
 
+    public static IRunProcessService get() {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
+            return (IRunProcessService) GlobalServiceRegister.getDefault().getService(IRunProcessService.class);
+        }
+        return null;
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/ProjectManager.java

@@ -20,7 +20,9 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.WeakHashMap;
 
+import org.apache.commons.lang.StringUtils;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.resources.ResourcesPlugin;
@@ -91,11 +93,16 @@ public final class ProjectManager {
 
     private Set<Object> updatedRemoteHandlerRecords;
 
+    private Set<Project> tempProjects;
+
+    private WeakHashMap<IRepositoryViewObject, Boolean> cachedObjects = new WeakHashMap<IRepositoryViewObject, Boolean>();
+
     private ProjectManager() {
         beforeLogonRecords = new HashSet<String>();
         logonRecords = new HashSet<String>();
         migrationRecords = new HashSet<String>();
         updatedRemoteHandlerRecords = new HashSet<Object>();
+        tempProjects = new HashSet<>();
         initCurrentProject();
     }
 
@@ -106,6 +113,18 @@ public final class ProjectManager {
         return singleton;
     }
 
+    public void clearTempProjects() {
+        tempProjects.clear();
+    }
+
+    public boolean removeTempProject(Project project) {
+        return tempProjects.remove(project);
+    }
+
+    public boolean addTempProject(Project project) {
+        return tempProjects.add(project);
+    }
+
     public Project getProjectFromProjectLabel(String label) {
         if (currentProject == null) {
             initCurrentProject();
@@ -119,6 +138,12 @@ public final class ProjectManager {
                 return project;
             }
         }
+        
+        for (Project project : tempProjects) {
+            if (StringUtils.equals(project.getLabel(), label)) {
+                return project;
+            }
+        }
 
         return null;
     }
@@ -136,6 +161,11 @@ public final class ProjectManager {
                 return project;
             }
         }
+        for (Project project : tempProjects) {
+            if (StringUtils.equals(project.getTechnicalLabel(), label)) {
+                return project;
+            }
+        }
 
         return null;
     }
@@ -156,7 +186,7 @@ public final class ProjectManager {
             Set<String> resolvedProjectLabels, boolean force) {
         if (p != null) {
             String parentBranch = ProjectManager.getInstance().getMainProjectBranch(p);
-            if (parentBranch != null || force) {
+            if ((!p.isLocal() || parentBranch != null) || force) {
                 resolvedProjectLabels.add(p.getTechnicalLabel());
                 for (ProjectReference pr : new Project(p).getProjectReferenceList()) {
                     if (ProjectManager.validReferenceProject(p, pr)
@@ -200,6 +230,10 @@ public final class ProjectManager {
      * return all the referenced projects of current project.
      */
     public List<Project> getAllReferencedProjects(boolean force) {
+        return getAllReferencedProjects(getCurrentProject(), force);
+    }
+
+    public List<Project> getAllReferencedProjects(Project targetProject, boolean force) {
         List<Project> allReferencedprojects = new ArrayList<Project>();
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IProxyRepositoryService.class)) {
             if (this.getCurrentProject() == null) {
@@ -212,7 +246,7 @@ public final class ProjectManager {
             IProxyRepositoryFactory factory = service.getProxyRepositoryFactory();
             if (factory != null) {
                 List<org.talend.core.model.properties.Project> rProjects = factory
-                        .getReferencedProjects(this.getCurrentProject());
+                        .getReferencedProjects(targetProject);
                 if (rProjects != null) {
                     for (org.talend.core.model.properties.Project p : rProjects) {
                         Project project = new Project(p);
@@ -420,9 +454,14 @@ public final class ProjectManager {
                     if (object == null) {
                         return true;
                     }
+                    if (cachedObjects.containsKey(object)) {
+                        return cachedObjects.get(object);
+                    }
                     org.talend.core.model.properties.Project emfProject = getProject(object.getProperty().getItem());
                     org.talend.core.model.properties.Project curProject = curP.getEmfProject();
-                    return emfProject.equals(curProject);
+                    boolean ret = emfProject.equals(curProject);
+                    cachedObjects.put(object, ret);
+                    return ret;
 
                 } else {
                     IProjectRepositoryNode root = node.getRoot();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/RepositoryWorkUnit.java

@@ -12,6 +12,8 @@
 // ============================================================================
 package org.talend.repository;
 
+import java.util.Set;
+
 import org.talend.commons.exception.LoginException;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.core.model.general.Project;
@@ -45,6 +47,8 @@ public abstract class RepositoryWorkUnit<T> {
 
     private boolean refreshRepository = true; // added for TDI-27085
 
+    private Set<String> specifiedCommitFiles;
+
     /**
      * Usefull for some save only actions, where we're sure everything is up to date.
      */
@@ -194,7 +198,14 @@ public abstract class RepositoryWorkUnit<T> {
         this.refreshRepository = refreshRepository;
     }
 
-    
+    public Set<String> getSpecifiedCommitFiles() {
+        return specifiedCommitFiles;
+    }
+
+    public void setSpecifiedCommitFiles(Set<String> specifiedCommitFiles) {
+        this.specifiedCommitFiles = specifiedCommitFiles;
+    }
+
     /**
      * Getter for avoidUpdateLocks.
      * @return the avoidUpdateLocks

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/model/RepositoryConstants.java

@@ -33,6 +33,8 @@ public class RepositoryConstants {
 
     public static final String PROJECT_BRANCH_ID = "repository.project.branch"; //$NON-NLS-1$
 
+    public static final String PROJECT_ID = "repository.project.id";
+
     public static final String IMG_DIRECTORY = "images"; //$NON-NLS-1$
 
     public static final String IMG_DIRECTORY_OF_JOB_OUTLINE = "images/job_outlines"; //$NON-NLS-1$

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/metadata/command/RepositoryChangeMetadataForSAPBapi.java

@@ -17,6 +17,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
 import org.eclipse.emf.common.util.EList;
 import org.eclipse.emf.common.util.EMap;
 import org.eclipse.gef.commands.Command;
@@ -152,8 +153,8 @@ public class RepositoryChangeMetadataForSAPBapi extends Command {
 
             String uinqueTableName = node.getProcess().generateUniqueConnectionName(
                     MultiSchemasUtil.getConnectionBaseName(newMetadatTable.getLabel()));
-            String paramType = getParamType(newMetadatTable, isInput);
-            if (paramType == null) {
+            String type = getType(newMetadatTable, isInput);
+            if (type == null) {
                 return;
             }
             if (selectionIndex != null && selectionIndex < paramValues.size()) {
@@ -165,18 +166,19 @@ public class RepositoryChangeMetadataForSAPBapi extends Command {
                 paramValues.add(valueMap);
             }
             valueMap.put(ISAPConstant.NAME, TalendQuoteUtils.addQuotes(newMetadatTable.getLabel()));
-            valueMap.put(ISAPConstant.TYPE, paramType);
+            valueMap.put(ISAPConstant.TYPE, type);
             valueMap.put(ISAPConstant.FIELD_SCHEMA, uinqueTableName);
             if (isInput) {
                 valueMap.put(ISAPConstant.PARENT_ROW, ""); //$NON-NLS-1$
-                if (functionUnit != null && functionUnit.getParamData() != null
-                        && functionUnit.getParamData().getInputRoot() != null) {
-                    for (SAPFunctionParameter param : functionUnit.getParamData().getInputRoot().getChildren()) {
-                        if (param.getName().equals(newMetadatTable.getTableName())) {
-                            valueMap.put(ISAPConstant.CHANGING, param.isChanging());
-                        }
-                    }
+
+                String inputParameterType = getInputParameterType(functionUnit, newMetadatTable);
+                if (StringUtils.isBlank(inputParameterType)) {
+                    inputParameterType = ISAPConstant.PARAMETER_TYPE_IMPORT;
                 }
+                valueMap.put(ISAPConstant.PARAMETER_TYPE, inputParameterType);
+
+                valueMap.put(ISAPConstant.CHANGING, ISAPConstant.PARAMETER_TYPE_CHANGING.equals(inputParameterType));
+
                 Map<String, String> properties = newMetadatTable.getAdditionalProperties();
                 if (properties != null) {
                     properties.put(ISINPUT, TRUE);
@@ -195,7 +197,37 @@ public class RepositoryChangeMetadataForSAPBapi extends Command {
         }
     }
 
-    private String getParamType(IMetadataTable table, boolean isInput) {
+    private String getInputParameterType(SAPFunctionUnit funcUnit, IMetadataTable newMetadatTable) {
+        String parameterType = "";
+        if (funcUnit != null && funcUnit.getParamData() != null && funcUnit.getParamData().getInputRoot() != null) {
+            SAPFunctionParameter foundParam = null;
+            String tableName = newMetadatTable.getTableName();
+            for (SAPFunctionParameter param : funcUnit.getParamData().getInputRoot().getChildren()) {
+                if (tableName.equals(param.getName())) {
+                    foundParam = param;
+                    break;
+                }
+            }
+            if (foundParam != null) {
+                if (foundParam.isChanging()) {
+                    parameterType = ISAPConstant.PARAMETER_TYPE_CHANGING;
+                } else {
+                    if (ISAPConstant.PARAM_TABLE.equals(foundParam.getType())) {
+                        if (foundParam.isTableResideInTables()) {
+                            parameterType = ISAPConstant.PARAMETER_TYPE_TABLES;
+                        } else {
+                            parameterType = ISAPConstant.PARAMETER_TYPE_IMPORT;
+                        }
+                    } else {
+                        parameterType = ISAPConstant.PARAMETER_TYPE_IMPORT;
+                    }
+                }
+            }
+        }
+        return parameterType;
+    }
+
+    private String getType(IMetadataTable table, boolean isInput) {
         if (functionUnit == null) {
             return null;
         }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/metadata/dialog/MetadataDialog.java

@@ -725,6 +725,8 @@ public class MetadataDialog extends Dialog {
         }
 
         IDesignerCoreService designerCoreService = CoreUIPlugin.getDefault().getDesignerCoreService();
+        updateTableOriginalColumns(outputTable);
+        updateTableOriginalColumns(inputTable);
         designerCoreService.setTraceFilterParameters(outputNode, outputTable, preOutputColumnSet, changedNameColumnsForOutput);
         designerCoreService.setTraceFilterParameters(inputNode, inputTable, preInputColumnSet, changedNameColumnsForInput);
         if (outputTable != null && inputTable != null) {
@@ -745,6 +747,18 @@ public class MetadataDialog extends Dialog {
             }
         }
     }
+    
+    private void updateTableOriginalColumns(IMetadataTable table) {
+        if (table != null && table.getListColumns() != null) {
+            List<String> columnNames = new ArrayList<String>();
+            for (IMetadataColumn column : table.getListColumns()) {
+                columnNames.add(column.getLabel());
+            }
+            if(table.getOriginalColumns() == null || table.getOriginalColumns().isEmpty()) {
+            	table.setOriginalColumns(columnNames);
+            }
+        }
+    }
 
     /**
      * DOC wzhang Comment method "getPreColumnsSet".

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/proposal/JavaSimpleDateFormatProposalProvider.java

@@ -72,10 +72,20 @@ public class JavaSimpleDateFormatProposalProvider implements IContentProposalPro
                         "\"dd/MM/yyyy\" : common format", "dd/MM/yyyy", "Date\n Examples : \n  01/01/2007\n  31/12/2007"), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
                 new DateFormatContentProposal(
                         "\"MM/dd/yyyy\" : common format", "MM/dd/yyyy", "Date\n Examples : \n  01/01/2007\n  12/31/2007"), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-                new DateFormatContentProposal("\"yyyy-MM-dd'T'HH:mm:ss\" : common format", "yyyy-MM-dd'T'HH:mm:ss", //$NON-NLS-1$ //$NON-NLS-2$
+                new DateFormatContentProposal(
+                        "\"yyyy-MM-dd\" : common format", "yyyy-MM-dd", "Date\n Examples : \n  2007-01-01\n  2007-12-31"), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+                new DateFormatContentProposal("\"yyyy-MM-dd HH:mm:ss\" : common format", "yyyy-MM-dd HH:mm:ss",
+                        "Date\n Examples : \n  2014-08-19 07:41:35\n  2014-12-31 07:41:35"),//$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+                new DateFormatContentProposal("\"yyyy-MM-dd'T'HH:mm:ss\" : common format", "yyyy-MM-dd'T'HH:mm:ss",
                         "Date\n Examples : \n  2007-01-01T00:00:00\n  2007-12-31T23:59:59"), // hywang //$NON-NLS-1$
                 new DateFormatContentProposal("\"yyyy-MM-dd'T'HH:mm:ss'000Z'\" : common format", "yyyy-MM-dd'T'HH:mm:ss'000Z'", //$NON-NLS-1$ //$NON-NLS-2$
                         "Date\n Examples : \n  2007-01-01T00:00:00000Z\n  2007-12-31T23:59:59000Z"), // hywang //$NON-NLS-1$
+                new DateFormatContentProposal("\"yyyy-MM-dd'T'HH:mm:ss.SSSSSS\" : common format", "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
+                        "Date\n Examples : \n  2019-04-18T09:47:49.568095"), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$       
+                new DateFormatContentProposal("\"yyyy-MM-dd HH:mm:ss zzz\" : common format", "yyyy-MM-dd HH:mm:ss zzz", //$NON-NLS-1$ //$NON-NLS-2$
+                        "Date\n Examples : \n  2008-01-01 00:00:00 UTC\n  2008-12-31 04:08:00 PDT"), 
+                new DateFormatContentProposal("\"yyyy-MM-dd HH:mm:ss.SSSXXX\" : common format", "yyyy-MM-dd HH:mm:ss.SSSXXX", //$NON-NLS-1$ //$NON-NLS-2$
+                        "Date\n Examples : \n  2014-08-19 07:41:35.220 -05:00"),
                 new DateFormatContentProposal(
                         "\"HH:mm:ss\" : common format", "HH:mm:ss", "Date\n Examples : \n  00:00:00 \n  23:59:59"), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
                 new DateFormatContentProposal("\"yy\" : Year (00-99)", "yy", "Year \nExamples : \n  98\n  07"), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

main/plugins/org.talend.core/src/main/java/org/talend/core/CoreService.java

@@ -54,6 +54,7 @@ import org.talend.core.model.metadata.QueryUtil;
 import org.talend.core.model.metadata.builder.ConvertionHelper;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.process.ElementParameterParser;
+import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.model.repository.IRepositoryViewObject;
@@ -279,9 +280,10 @@ public class CoreService implements ICoreService {
             ICodeGeneratorService codeGenService = (ICodeGeneratorService) GlobalServiceRegister.getDefault().getService(
                     ICodeGeneratorService.class);
             codeGenService.createRoutineSynchronizer().syncAllRoutinesForLogOn();
-            codeGenService.createRoutineSynchronizer().syncAllPigudfForLogOn();
+            if (ProcessUtils.isRequiredPigUDFs(null)) {
+                codeGenService.createRoutineSynchronizer().syncAllPigudfForLogOn();
+            }
         }
-
     }
 
     @Override

main/plugins/org.talend.core/src/main/java/org/talend/core/model/utils/ProcessStreamTrashReaderUtil.java

@@ -16,7 +16,7 @@ import java.io.BufferedReader;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 
-import org.talend.commons.exception.WarningException;
+import org.talend.commons.exception.ExceptionHandler;
 
 /**
  * cli class global comment. Detailled comment
@@ -104,26 +104,31 @@ public final class ProcessStreamTrashReaderUtil {
 
     public static void readAndForget(final Process process) {
         try {
+            final String encoding = "UTF-8";
             // input stream thread
             new Thread() {
 
                 public void run() {
-                    InputStream is = process.getInputStream();
-                    InputStreamReader din = new InputStreamReader(is);
-                    BufferedReader reader = new BufferedReader(din);
                     try {
-                        String line = null;
-                        while ((line = reader.readLine()) != null) {
-                            System.out.println("getInputStream " + line); //$NON-NLS-1$
-                        }
-                    } catch (Exception ex) {
-                        // nothing to do
-                    } finally {
+                        InputStream is = process.getInputStream();
+                        InputStreamReader din = new InputStreamReader(is, encoding);
+                        BufferedReader reader = new BufferedReader(din);
                         try {
-                            is.close();
+                            String line = null;
+                            while ((line = reader.readLine()) != null) {
+                                System.out.println("getInputStream " + line); //$NON-NLS-1$
+                            }
                         } catch (Exception ex) {
                             // nothing to do
+                        } finally {
+                            try {
+                                is.close();
+                            } catch (Exception ex) {
+                                // nothing to do
+                            }
                         }
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
                     }
                 }
             }.start();
@@ -132,23 +137,31 @@ public final class ProcessStreamTrashReaderUtil {
             new Thread() {
 
                 public void run() {
-                    InputStream is = process.getErrorStream();
-                    InputStreamReader din = new InputStreamReader(is);
-                    BufferedReader reader = new BufferedReader(din);
                     try {
-                        String line = null;
-                        while ((line = reader.readLine()) != null) {
-                            System.out.println("getErrorStream " + line); //$NON-NLS-1$
-                            throw new WarningException("AAAAA");
-                        }
-                    } catch (Exception ex) {
-                        // nothing to do
-                    } finally {
+                        InputStream is = process.getErrorStream();
+                        InputStreamReader din = new InputStreamReader(is, encoding);
+                        BufferedReader reader = new BufferedReader(din);
+                        StringBuilder strBuilder = new StringBuilder();
                         try {
-                            is.close();
+                            String line = null;
+                            while ((line = reader.readLine()) != null) {
+                                System.out.println("getErrorStream " + line); //$NON-NLS-1$
+                                strBuilder.append(line).append("\n");
+                            }
                         } catch (Exception ex) {
                             // nothing to do
+                        } finally {
+                            try {
+                                is.close();
+                            } catch (Exception ex) {
+                                // nothing to do
+                            }
                         }
+                        if (!strBuilder.toString().replaceAll("[\n]", " ").trim().isEmpty()) {
+                            throw new Exception(strBuilder.toString());
+                        }
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
                     }
                 }
             }.start();
@@ -204,7 +217,7 @@ public final class ProcessStreamTrashReaderUtil {
                         String line = null;
                         while ((line = reader.readLine()) != null) {
                             System.out.println("getErrorStream " + line); //$NON-NLS-1$
-                            if(line.startsWith("log4j:WARN")){
+                            if(line.startsWith("log4j:WARN") || line.startsWith("SLF4J:")){
                                 continue;
                             }
                             buffer.append(line);

main/plugins/org.talend.core/src/main/java/org/talend/core/services/ICoreTisService.java

@@ -19,11 +19,14 @@ import org.eclipse.emf.common.util.URI;
 import org.osgi.service.prefs.BackingStoreException;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.exception.SystemException;
+import org.talend.core.GlobalServiceRegister;
 import org.talend.core.IService;
 import org.talend.core.model.general.Project;
 import org.talend.core.model.process.INode;
+import org.talend.core.model.properties.Property;
 import org.talend.core.updatesite.IUpdateSiteBean;
 
+
 /**
  * wchen class global comment. Detailled comment
  */
@@ -62,4 +65,17 @@ public interface ICoreTisService extends IService {
     public File getLicenseFile();
     
     public String generateSignerSessionId();
+
+    public void afterImport (Property property) throws PersistenceException;  
+
+    void updateSignature();
+
+    public static ICoreTisService get() {
+        GlobalServiceRegister register = GlobalServiceRegister.getDefault();
+        if (register.isServiceRegistered(ICoreTisService.class)) {
+            return (ICoreTisService) register.getService(ICoreTisService.class);
+        }
+        return null;
+    }
+
 }

main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java

@@ -167,6 +167,8 @@ public class ProcessorUtilities {
 
     private static boolean isDebug = false;
 
+    private static JobInfo mainJobInfo;
+
     public static void addOpenEditor(IEditorPart editor) {
         openedEditors.add(editor);
     }
@@ -435,6 +437,7 @@ public class ProcessorUtilities {
 
             isMainJob = true;
             codeModified = false;
+            mainJobInfo = jobInfo;
 
             // this cache only keep the last main job's generation, so clear it since we regenerate a new job.
             LastGenerationInfo.getInstance().getLastGeneratedjobs().clear();
@@ -507,7 +510,7 @@ public class ProcessorUtilities {
             mainRelation.setId(jobInfo.getJobId());
             mainRelation.setVersion(jobInfo.getJobVersion());
             mainRelation.setType(RelationshipItemBuilder.JOB_RELATION);
-            hasLoopDependency = checkLoopDependencies(mainRelation);
+            hasLoopDependency = checkLoopDependencies(mainRelation, new HashMap<String, String>());
             // clean the previous code in case it has deleted subjob
             cleanSourceFolder(progressMonitor, currentProcess, processor);
         }
@@ -607,28 +610,29 @@ public class ProcessorUtilities {
         return processor;
     }
 
-    private static boolean checkLoopDependencies(Relation mainJobInfo) throws ProcessorException {
-        List<Relation> itemsJobRelatedTo = getItemsRelation(mainJobInfo);
+    private static boolean checkLoopDependencies(Relation mainJobInfo, Map<String, String> idToLastestVersionMap)
+            throws ProcessorException {
+        List<Relation> itemsJobRelatedTo = getItemsRelation(mainJobInfo, idToLastestVersionMap);
         List<Relation> relationChecked = new ArrayList<>();
         relationChecked.add(mainJobInfo);
-        return checkLoopDependencies(mainJobInfo, itemsJobRelatedTo, relationChecked);
+        return checkLoopDependencies(mainJobInfo, itemsJobRelatedTo, relationChecked, idToLastestVersionMap);
     }
 
     private static boolean checkLoopDependencies(Relation mainRelation, List<Relation> itemsJobRelatedTo,
-            List<Relation> relationChecked) throws ProcessorException {
+            List<Relation> relationChecked, Map<String, String> idToLastestVersionMap) throws ProcessorException {
         boolean hasDependency = false;
         for (Relation relation : itemsJobRelatedTo) {
             hasDependency = relation.getId().equals(mainRelation.getId())
                     && relation.getVersion().equals(mainRelation.getVersion());
             if (!hasDependency) {
-                List<Relation> itemsChildJob = getItemsRelation(relation);
+                List<Relation> itemsChildJob = getItemsRelation(relation, idToLastestVersionMap);
                 if (!relationChecked.contains(relation)) {
                     relationChecked.add(relation);
-                    hasDependency = checkLoopDependencies(mainRelation, itemsChildJob, relationChecked);
+                    hasDependency = checkLoopDependencies(mainRelation, itemsChildJob, relationChecked, idToLastestVersionMap);
                 }
                 if (!hasDependency) {
                     for (Relation childRelation : itemsChildJob) {
-                        hasDependency = checkLoopDependencies(childRelation);
+                        hasDependency = checkLoopDependencies(childRelation, idToLastestVersionMap);
                         if (hasDependency) {
                             break;
                         }
@@ -643,7 +647,7 @@ public class ProcessorUtilities {
         return hasDependency;
     }
 
-    private static List<Relation> getItemsRelation(Relation mainJobInfo) throws ProcessorException {
+    private static List<Relation> getItemsRelation(Relation mainJobInfo, Map<String, String> idToLastestVersionMap) throws ProcessorException {
         List<Relation> itemsJobRelatedTo = new ArrayList<Relation>();
         try {
             List<Project> allProjects = new ArrayList<Project>();
@@ -657,12 +661,17 @@ public class ProcessorUtilities {
                         mainJobInfo.getType(), RelationshipItemBuilder.JOBLET_RELATION));
                 for (Relation relation : itemsJobRelatedTo) {
                     if (relation.getVersion().equals(RelationshipItemBuilder.LATEST_VERSION)) {
-                        for (Project project : allProjects) {
-                            IRepositoryViewObject lastVersion =
-                                    ProxyRepositoryFactory.getInstance().getLastVersion(project, relation.getId());
-                            if (lastVersion != null) {
-                                relation.setVersion(lastVersion.getVersion());
-                                break;
+                        if (idToLastestVersionMap.containsKey(relation.getId())) {
+                            relation.setVersion(idToLastestVersionMap.get(relation.getId()));
+                        } else {
+                            for (Project project : allProjects) {
+                                IRepositoryViewObject lastVersion =
+                                        ProxyRepositoryFactory.getInstance().getLastVersion(project, relation.getId());
+                                if (lastVersion != null) {
+                                    relation.setVersion(lastVersion.getVersion());
+                                    idToLastestVersionMap.put(relation.getId(), relation.getVersion());
+                                    break;
+                                }
                             }
                         }
                     }
@@ -950,6 +959,7 @@ public class ProcessorUtilities {
             if (jobInfo.getFatherJobInfo() == null) {
                 isMainJob = true;
                 codeModified = false;
+                mainJobInfo = jobInfo;
 
                 // this cache only keep the last main job's generation, so clear it since we regenerate a new job.
                 LastGenerationInfo.getInstance().getLastGeneratedjobs().clear();
@@ -1017,7 +1027,7 @@ public class ProcessorUtilities {
                 mainRelation.setId(jobInfo.getJobId());
                 mainRelation.setVersion(jobInfo.getJobVersion());
                 mainRelation.setType(RelationshipItemBuilder.JOB_RELATION);
-                hasLoopDependency = checkLoopDependencies(mainRelation);
+                hasLoopDependency = checkLoopDependencies(mainRelation, new HashMap<String, String>());
                 // clean the previous code in case it has deleted subjob
                 cleanSourceFolder(progressMonitor, currentProcess, processor);
             }
@@ -1184,10 +1194,9 @@ public class ProcessorUtilities {
             IFolder srcFolder = processor.getTalendJavaProject().getProject().getFolder(codePath);
             String jobPackageFolder = JavaResourcesHelper.getJobClassPackageFolder(currentProcess);
             for (IResource resource : srcFolder.members()) {
-                if (resource.getProjectRelativePath().toPortableString().endsWith(jobPackageFolder)) {
-                    break;
+                if (!resource.getProjectRelativePath().toPortableString().endsWith(jobPackageFolder)) {
+                    resource.delete(true, progressMonitor);
                 }
-                resource.delete(true, progressMonitor);
             }
         } catch (CoreException e) {
             ExceptionHandler.process(e);
@@ -1599,10 +1608,12 @@ public class ProcessorUtilities {
         jobList.clear();
         esbJobs.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         JobInfo jobInfo = new JobInfo(processName, contextName, version);
         IProcessor process = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
         jobList.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         return process;
     }
 
@@ -1625,9 +1636,11 @@ public class ProcessorUtilities {
         jobList.clear();
         esbJobs.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         IProcessor process = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
         jobList.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         return process;
     }
 
@@ -1645,9 +1658,11 @@ public class ProcessorUtilities {
         jobList.clear();
         esbJobs.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         IProcessor result = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
         jobList.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         return result;
     }
 
@@ -1665,10 +1680,12 @@ public class ProcessorUtilities {
         jobList.clear();
         esbJobs.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         IProcessor result =
                 generateCode(jobInfo, contextName, statistics, trace, needContext, GENERATE_ALL_CHILDS, monitor);
         jobList.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         return result;
     }
 
@@ -1695,9 +1712,11 @@ public class ProcessorUtilities {
         jobList.clear();
         esbJobs.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         IProcessor result = generateCode(jobInfo, contextName, statistics, trace, needContext, option, monitor);
         jobList.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         return result;
     }
 
@@ -1718,9 +1737,11 @@ public class ProcessorUtilities {
             jobList.clear();
             esbJobs.clear();
             hasLoopDependency = false;
+            mainJobInfo = null;
             result = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
             jobList.clear();
             hasLoopDependency = false;
+            mainJobInfo = null;
         }
         return result;
     }
@@ -1730,9 +1751,11 @@ public class ProcessorUtilities {
         jobList.clear();
         esbJobs.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         IProcessor returnValue = generateCode(process, contextName, statistics, trace, false);
         jobList.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         return returnValue;
     }
 
@@ -1764,10 +1787,12 @@ public class ProcessorUtilities {
         jobList.clear();
         esbJobs.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         IProcessor genCode = generateCode(jobInfo, context.getName(), statistics, trace, contextProperties,
                 GENERATE_ALL_CHILDS, new NullProgressMonitor());
         jobList.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         return genCode;
     }
 
@@ -1776,10 +1801,12 @@ public class ProcessorUtilities {
         jobList.clear();
         esbJobs.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         IProcessor returnValue =
                 generateCode(process, context, statistics, trace, properties, new NullProgressMonitor());
         jobList.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         return returnValue;
     }
 
@@ -1811,10 +1838,12 @@ public class ProcessorUtilities {
         jobList.clear();
         esbJobs.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         IProcessor genCode = generateCode(jobInfo, context.getName(), statistics, trace, properties,
                 GENERATE_ALL_CHILDS, progressMonitor);
         jobList.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         return genCode;
     }
 
@@ -1860,10 +1889,12 @@ public class ProcessorUtilities {
         jobList.clear();
         esbJobs.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         IProcessor genCode = generateCode(processor, jobInfo, context.getName(), statistics, trace, properties,
                 GENERATE_ALL_CHILDS, progressMonitor);
         jobList.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
 
         TimeMeasure.end(timeMeasureGenerateCodesId);
         // if active before, not disable and active still.
@@ -1890,10 +1921,12 @@ public class ProcessorUtilities {
         jobList.clear();
         esbJobs.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         IProcessor genCode = generateCode(jobInfo, context.getName(), statistics, trace, properties, option,
                 new NullProgressMonitor());
         jobList.clear();
         hasLoopDependency = false;
+        mainJobInfo = null;
         return genCode;
     }
 
@@ -2150,7 +2183,7 @@ public class ProcessorUtilities {
         EList<NodeType> nodes = ptype.getNode();
         String jobletPaletteType = null;
         String frameWork = ptype.getFramework();
-        if (frameWork == null) {
+        if (StringUtils.isBlank(frameWork)) {
             jobletPaletteType = ComponentCategory.CATEGORY_4_DI.getName();
         } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK)) {
             jobletPaletteType = ComponentCategory.CATEGORY_4_SPARK.getName();
@@ -2195,12 +2228,14 @@ public class ProcessorUtilities {
                         ProcessItem processItem = ItemCacheManager.getProcessItem(jobId, jobVersion);
                         if (processItem != null) {
                             JobInfo jobInfo = new JobInfo(processItem, jobContext);
-                            if (!jobInfos.contains(jobInfo)) {
-                                jobInfos.add(jobInfo);
-                                jobInfo.setFatherJobInfo(parentJobInfo);
-                                if (!firstChildOnly) {
-                                    getAllJobInfo(processItem.getProcess(), jobInfo, jobInfos, firstChildOnly);
-                                }
+                            jobInfos.add(jobInfo);
+                            JobInfo jobInfoFromSet = findJobInfo(jobInfos, jobInfo);
+                            boolean hasLoop = hasLoop(jobInfoFromSet);
+                            
+                            jobInfo.setFatherJobInfo(parentJobInfo);
+                            jobInfoFromSet.setFatherJobInfo(parentJobInfo);
+                            if (!firstChildOnly && !hasLoop) {
+                                getAllJobInfo(processItem.getProcess(), jobInfo, jobInfos, firstChildOnly);
                             }
                         }
                     }
@@ -2281,7 +2316,35 @@ public class ProcessorUtilities {
         return jobInfos;
     }
 
-    private static boolean isRouteletNode(NodeType node) {
+    private static JobInfo findJobInfo(Set<JobInfo> jobInfos, JobInfo jobInfoToFind) {
+        for (JobInfo jobInfo : jobInfos) {
+            if (jobInfo.equals(jobInfoToFind)) {
+                return jobInfo;
+            }
+        }
+        return null;
+    }
+
+    private static boolean hasLoop(JobInfo jobInfo) {
+
+        Set<String> jobInfoIds = new HashSet<String>();
+
+        JobInfo jobInfoCurr = jobInfo;
+
+        while(jobInfoCurr != null) {
+            if (jobInfoIds.contains(jobInfoCurr.getJobId())) {
+                return true;
+            }
+
+            jobInfoIds.add(jobInfoCurr.getJobId());
+
+            jobInfoCurr = jobInfoCurr.getFatherJobInfo();
+        }
+
+        return false;
+    }
+
+	private static boolean isRouteletNode(NodeType node) {
         String jobIds = getParameterValue(node.getElementParameter(), "PROCESS_TYPE:PROCESS_TYPE_PROCESS");
         String jobVersion = getParameterValue(node.getElementParameter(), "PROCESS_TYPE:PROCESS_TYPE_VERSION"); //$NON-NLS-1$
         ProcessItem processItem = ItemCacheManager.getProcessItem(jobIds, jobVersion);
@@ -2436,6 +2499,14 @@ public class ProcessorUtilities {
         return hasLoopDependency;
     }
 
+    /**
+     * Getter for mainJobInfo. <font color="red">Need to check null</font>
+     * 
+     * @return the mainJobInfo
+     */
+    public static JobInfo getMainJobInfo() {
+        return mainJobInfo;
+    }
     /**
      * The dynamic loading of the hadoop configuration library is supported in DI, MapReduce and Spark (batch and
      * streaming).
@@ -2467,15 +2538,44 @@ public class ProcessorUtilities {
         return doSupportDynamicHadoopConfLoading(property) && !isExportAsOSGI();
     }
 
+    public static boolean isEsbJob(IProcess process) {
+        return isEsbJob(process, false);
+    }
+    
     public static boolean isEsbJob(String processId, String version) {
         return esbJobs.contains(esbJobKey(processId, version));
     }
+    
+    public static boolean isEsbJob(IProcess process, boolean checkCurrentProcess) {
 
-    private static void addEsbJob(JobInfo jobInfo) {
-        if (esbJobs.contains(esbJobKey(jobInfo.getJobId(), jobInfo.getJobVersion()))) {
-            return;
+        if (process instanceof IProcess2) {
+
+            if (checkCurrentProcess) {
+                for (INode n : process.getGraphicalNodes()) {
+                    if (isEsbComponentName(n.getComponent().getName())) {
+                        return true;
+                    }
+                }
+            } else {
+                Set<JobInfo> infos = ProcessorUtilities.getChildrenJobInfo(((IProcess2) process).getProperty().getItem(), false);
+
+                for (JobInfo jobInfo : infos) {
+                    ProcessType processType = jobInfo.getProcessItem().getProcess();
+                    EList<NodeType> nodes = processType.getNode();
+                    for (NodeType nodeType : nodes) {
+                        if (isEsbComponentName(nodeType.getComponentName())) {
+                            return true;
+                        }
+                    }
+                }
+            }
+            return false;
         }
 
+        return false;
+    }
+
+    private static void addEsbJob(JobInfo jobInfo) {
         esbJobs.add(esbJobKey(jobInfo.getJobId(), jobInfo.getJobVersion()));
         if (jobInfo.getFatherJobInfo() != null) {
             addEsbJob(jobInfo.getFatherJobInfo());

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/RepositorySystemFactory.java

@@ -35,6 +35,7 @@ import org.eclipse.aether.util.artifact.SubArtifact;
 import org.eclipse.aether.util.listener.ChainedRepositoryListener;
 import org.eclipse.aether.util.listener.ChainedTransferListener;
 import org.eclipse.aether.util.repository.AuthenticationBuilder;
+import org.talend.designer.maven.aether.util.TalendAetherProxySelector;
 
 /**
  * created by wchen on Aug 10, 2017 Detailled comment
@@ -72,6 +73,8 @@ public class RepositorySystemFactory {
                     .setLocalRepositoryManager(system.newLocalRepositoryManager(repositorySystemSession, localRepo));
             repositorySystemSession.setTransferListener(new ChainedTransferListener());
             repositorySystemSession.setRepositoryListener(new ChainedRepositoryListener());
+            repositorySystemSession.setProxySelector(new TalendAetherProxySelector());
+            sessions.put(localRepo, repositorySystemSession);
         }
 
         return repositorySystemSession;
@@ -107,6 +110,7 @@ public class RepositorySystemFactory {
         Authentication auth = new AuthenticationBuilder().addUsername(userName).addPassword(password).build();
         RemoteRepository distRepo = new RemoteRepository.Builder(repositoryId, "default", repositoryUrl).setAuthentication(auth)
                 .build();
+        distRepo = new RemoteRepository.Builder(distRepo).setProxy(new TalendAetherProxySelector().getProxy(distRepo)).build();
 
         deployRequest.setRepository(distRepo);
 

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/DynamicDistributionAetherUtils.java

@@ -129,6 +129,7 @@ public class DynamicDistributionAetherUtils {
             builder = builder.setAuthentication(auth);
         }
         RemoteRepository central = builder.build();
+        central = new RemoteRepository.Builder(central).setProxy(new TalendAetherProxySelector().getProxy(central)).build();
 
         CollectRequest collectRequest = new CollectRequest();
         collectRequest.setRoot(dependency);
@@ -180,6 +181,7 @@ public class DynamicDistributionAetherUtils {
             builder = builder.setAuthentication(auth);
         }
         RemoteRepository central = builder.build();
+        central = new RemoteRepository.Builder(central).setProxy(new TalendAetherProxySelector().getProxy(central)).build();
 
         VersionRangeRequest verRangeRequest = new VersionRangeRequest();
         verRangeRequest.addRepository(central);
@@ -226,6 +228,7 @@ public class DynamicDistributionAetherUtils {
             builder = builder.setAuthentication(auth);
         }
         RemoteRepository central = builder.build();
+        central = new RemoteRepository.Builder(central).setProxy(new TalendAetherProxySelector().getProxy(central)).build();
 
         VersionRangeRequest verRangeRequest = new VersionRangeRequest();
         verRangeRequest.addRepository(central);
@@ -299,6 +302,7 @@ public class DynamicDistributionAetherUtils {
 
         LocalRepository localRepo = new LocalRepository(repositoryPath);
         session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo));
+        session.setProxySelector(new TalendAetherProxySelector());
 
         updateDependencySelector(session, monitor);
 

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/TalendAetherProxySelector.java

@@ -0,0 +1,98 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.designer.maven.aether.util;
+
+import org.eclipse.aether.repository.Authentication;
+import org.eclipse.aether.repository.Proxy;
+import org.eclipse.aether.repository.ProxySelector;
+import org.eclipse.aether.repository.RemoteRepository;
+import org.eclipse.aether.util.repository.AuthenticationBuilder;
+import org.eclipse.aether.util.repository.DefaultProxySelector;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.TalendProxySelector;
+
+
+/**
+ * DOC cmeng  class global comment. Detailled comment
+ */
+public class TalendAetherProxySelector implements ProxySelector {
+
+    private boolean isTalendDebug = false;
+
+    public TalendAetherProxySelector() {
+        isTalendDebug = Boolean.valueOf(System.getProperty(TalendProxySelector.PROP_PRINT_LOGS, "false"));
+    }
+
+    @Override
+    public Proxy getProxy(RemoteRepository repository) {
+        /**
+         * Update each time in case the settings are changed
+         */
+        Proxy proxy = createProxySelector().getProxy(repository);
+        if (isTalendDebug) {
+            try {
+                if (repository != null) {
+                    String proxyStr = "";
+                    if (proxy != null) {
+                        proxyStr = proxy.getType() + " " + proxy.toString() + ", proxy user: "
+                                + (proxy.getAuthentication() != null ? "..." : "<empty>");
+                    }
+                    ExceptionHandler.log("Aether proxy> host: " + repository.getHost() + ", proxy: " + proxyStr);
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+        }
+        return proxy;
+    }
+
+    private DefaultProxySelector createProxySelector() {
+        DefaultProxySelector proxySelector = new DefaultProxySelector();
+        javaDefaultProxy(proxySelector);
+        return proxySelector;
+    }
+
+    private void javaDefaultProxy(DefaultProxySelector proxySelector) {
+        String[] schemas = new String[] { "http", "https" };
+        for (String schema : schemas) {
+            Proxy proxy = createProxy(schema);
+            if (proxy != null) {
+                proxySelector.add(proxy, System.getProperty(schema + ".nonProxyHosts"));
+            }
+        }
+    }
+
+    private org.eclipse.aether.repository.Proxy createProxy(String schema) {
+        String proxyHost = System.getProperty(schema + ".proxyHost");
+        if (proxyHost == null) {
+            return null;
+        }
+        String proxyUser = System.getProperty(schema + ".proxyUser");
+        String proxyPassword = System.getProperty(schema + ".proxyPassword");
+        int proxyPort = Integer.parseInt(System.getProperty(schema + ".proxyPort", "8080"));
+
+        Authentication authentication = createAuthentication(proxyUser, proxyPassword);
+        org.eclipse.aether.repository.Proxy proxyObj = new org.eclipse.aether.repository.Proxy(schema, proxyHost, proxyPort,
+                authentication);
+        return proxyObj;
+    }
+
+    private Authentication createAuthentication(String proxyUser, String proxyPassword) {
+        Authentication authentication = null;
+        if (proxyUser != null) {
+            authentication = new AuthenticationBuilder().addUsername(proxyUser).addPassword(proxyPassword).build();
+        }
+        return authentication;
+    }
+
+}

main/plugins/org.talend.designer.maven.ui/src/main/java/messages.properties

@@ -11,4 +11,7 @@ ProjectPomProjectSettingPage_FilterPomLabel=Filter to use to generate poms:
 ProjectPomProjectSettingPage_FilterErrorMessage=Invalid filter: {0}
 ProjectPomProjectSettingPage.syncAllPomsButtonText=Force full re-synchronize poms
 AbstractPersistentProjectSettingPage.syncAllPoms=Do you want to update all poms? \n This operation might take long time depends on your project size.
-MavenProjectSettingPage.filterExampleMessage=Filter examples:\nlabel=myJob       \t\t\t\t=> Generate only the job named "myJob"\n!(label=myJob)    \t\t\t\t=> Generate any job except the one named "myJob"\n(path=folder1/folder2)  \t\t\t=> Generate any job in the folder "folder1/folder2"\n(path=folder1/folder2)or(label=myJob)\t=> Generate any job in the folder "folder1/folder2" or named "myJob"\n(label=myJob)and(version=0.2)	\t=> Generate only the job named "myJob" with version 0.2\n!((label=myJob)and(version=0.1))	\t=> Generate every jobs except the "myJob" version 0.1
+MavenProjectSettingPage.filterExampleMessage=Filter examples:\nlabel=myJob       \t\t\t\t=> Generate only the job named "myJob"\n!(label=myJob)    \t\t\t\t=> Generate any job except the one named "myJob"\n(path=folder1/folder2)  \t\t\t=> Generate any job in the folder "folder1/folder2"\n(path=folder1/folder2)or(label=myJob)\t=> Generate any job in the folder "folder1/folder2" or named "myJob"\n(label=myJob)and(version=0.2)	\t=> Generate only the job named "myJob" with version 0.2\n!((label=myJob)and(version=0.1))	\t=> Generate every jobs except the "myJob" version 0.1
+MavenProjectSettingPage.refModuleText=Set reference project modules in profile
+MavenProjectSettingPage.syncAllPomsWarning=Click the Force full re-synchronize poms button to apply the new settings.
+MavenProjectSettingPage.skipFolders=Skip folders 

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/MavenUIService.java

@@ -14,6 +14,7 @@ package org.talend.designer.maven.ui;
 
 import java.util.Dictionary;
 import java.util.Hashtable;
+import java.util.Set;
 
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IProgressMonitor;
@@ -118,6 +119,17 @@ public class MavenUIService implements IMavenUIService {
         String studioUserSettingsFile = MavenPlugin.getMavenConfiguration().getUserSettingsFile();
         // apply the user settings to MavenResolver
         Dictionary<String, String> props = new Hashtable<String, String>();
+        Set<Object> keySet = System.getProperties().keySet();
+        if (keySet != null) {
+            for (Object keyObj : keySet) {
+                if (keyObj instanceof String) {
+                    String key = keyObj.toString();
+                    if (key.startsWith("org.ops4j.pax.url.mvn.")) {
+                        props.put(key, System.getProperty(key));
+                    }
+                }
+            }
+        }
         if (studioUserSettingsFile != null && !"".equals(studioUserSettingsFile)) {
             // change back to use the user settings after Pax-url-eather fix the space bug
             props.put("org.ops4j.pax.url.mvn.settings", studioUserSettingsFile);

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/setting/preference/M2eUserSettingForTalendLoginTask.java

@@ -46,6 +46,7 @@ import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.utils.VersionUtils;
 import org.talend.commons.utils.time.TimeMeasure;
 import org.talend.core.runtime.projectsetting.IProjectSettingTemplateConstants;
+import org.talend.core.services.ICoreTisService;
 import org.talend.designer.maven.DesignerMavenPlugin;
 import org.talend.designer.maven.repository.DefaultMavenRepositoryProvider;
 import org.talend.designer.maven.template.MavenTemplateManager;
@@ -229,6 +230,14 @@ public class M2eUserSettingForTalendLoginTask extends AbstractLoginTask {
         } catch (Exception e) {
             ExceptionHandler.process(e);
         } finally {
+            try {
+                ICoreTisService tisService = ICoreTisService.get();
+                if (tisService != null) {
+                    tisService.updateSignature();
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
             TimeMeasure.end("M2eUserSettingForTalendLoginTask"); //$NON-NLS-1$
             TimeMeasure.display = false;
             TimeMeasure.displaySteps = false;

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/setting/project/page/MavenProjectSettingPage.java

@@ -25,6 +25,7 @@ import org.eclipse.swt.widgets.Button;
 import org.eclipse.swt.widgets.Composite;
 import org.eclipse.swt.widgets.Label;
 import org.eclipse.swt.widgets.Text;
+import org.eclipse.ui.forms.IMessage;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.PluginChecker;
@@ -46,6 +47,10 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
 
 	private IPreferenceStore preferenceStore;
 
+    private Button useProfileModuleCheckbox;
+
+    private Button skipFoldersCheckbox;
+
 	public MavenProjectSettingPage() {
 		noDefaultAndApplyButton();
 	}
@@ -79,6 +84,28 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
         filterExampleLable.setText(Messages.getString("MavenProjectSettingPage.filterExampleMessage")); //$NON-NLS-1$
         filterExampleLable.setLayoutData(new GridData(SWT.FILL, SWT.CENTER, true, false));
 
+        useProfileModuleCheckbox = new Button(parent, SWT.CHECK);
+        useProfileModuleCheckbox.setText(Messages.getString("MavenProjectSettingPage.refModuleText")); //$NON-NLS-1$
+        useProfileModuleCheckbox.setSelection(preferenceStore.getBoolean(MavenConstants.USE_PROFILE_MODULE));
+        useProfileModuleCheckbox.addSelectionListener(new SelectionAdapter() {
+
+            @Override
+            public void widgetSelected(SelectionEvent e) {
+                addSyncWarning();
+            }
+        });
+
+        skipFoldersCheckbox = new Button(parent, SWT.CHECK);
+        skipFoldersCheckbox.setText(Messages.getString("MavenProjectSettingPage.skipFolders")); //$NON-NLS-1$
+        skipFoldersCheckbox.setSelection(preferenceStore.getBoolean(MavenConstants.SKIP_FOLDERS));
+        skipFoldersCheckbox.addSelectionListener(new SelectionAdapter() {
+
+            @Override
+            public void widgetSelected(SelectionEvent e) {
+                addSyncWarning();
+            }
+        });
+
         filterText.setText(filter);
 		filterText.addModifyListener(new ModifyListener() {
 
@@ -109,6 +136,7 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
 			public void widgetSelected(SelectionEvent event) {
 				try {
 					preferenceStore.setValue(MavenConstants.POM_FILTER, filter);
+					preferenceStore.setValue(MavenConstants.USE_PROFILE_MODULE, useProfileModuleCheckbox.getSelection());
 					new AggregatorPomsHelper().syncAllPoms();
 				} catch (Exception e) {
 					ExceptionHandler.process(e);
@@ -122,11 +150,17 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
 
 	}
 
+    private void addSyncWarning() {
+        setMessage(Messages.getString("MavenProjectSettingPage.syncAllPomsWarning"), IMessage.WARNING); //$NON-NLS-1$
+    }
+
 	@Override
 	public boolean performOk() {
 		boolean ok = super.performOk();
 		if (preferenceStore != null) {
 			preferenceStore.setValue(MavenConstants.POM_FILTER, filter);
+			preferenceStore.setValue(MavenConstants.USE_PROFILE_MODULE, useProfileModuleCheckbox.getSelection());
+            preferenceStore.setValue(MavenConstants.SKIP_FOLDERS, skipFoldersCheckbox.getSelection());
 		}
 		return ok;
 	}

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/launch/MavenCommandLauncher.java

@@ -393,8 +393,13 @@ public abstract class MavenCommandLauncher {
         public void waitFinish(ILaunch launch) {
 
             try {
+                boolean isCommandLine = CommonsPlugin.isHeadless();
                 while (!launchFinished) {
-                    Thread.sleep(100);
+                    if (isCommandLine) {
+                        Thread.sleep(100);
+                    } else {
+                        waitStudioFinish();
+                    }
                     // if terminated also
                     if (launch.getProcesses() != null && launch.getProcesses().length > 0) {
                         if (launch.getProcesses()[0].isTerminated()) {
@@ -409,5 +414,16 @@ public abstract class MavenCommandLauncher {
                 ExceptionHandler.process(e);
             }
         }
+
+        private void waitStudioFinish() throws InterruptedException {
+            org.eclipse.swt.widgets.Display display = org.eclipse.swt.widgets.Display.getCurrent();
+            if (display != null) {
+                if (!display.readAndDispatch()) {
+                    display.sleep();
+                }
+            } else {
+                Thread.sleep(100);
+            }
+        }
     }
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/AggregatorPomsHelper.java

@@ -12,17 +12,29 @@
 // ============================================================================
 package org.talend.designer.maven.tools;
 
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.*;
+import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_AGGREGATORS;
+import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_BEANS;
+import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_CODES;
+import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_JOBS;
+import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_PIGUDFS;
+import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_POMS;
+import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_ROUTINES;
 
 import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
-import java.util.ListIterator;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.maven.model.Activation;
 import org.apache.maven.model.Model;
+import org.apache.maven.model.Profile;
 import org.eclipse.core.resources.IContainer;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
@@ -49,7 +61,6 @@ import org.talend.core.context.Context;
 import org.talend.core.context.RepositoryContext;
 import org.talend.core.model.general.ILibrariesService;
 import org.talend.core.model.general.ModuleNeeded;
-import org.talend.core.model.general.Project;
 import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.ProjectReference;
@@ -70,6 +81,7 @@ import org.talend.core.runtime.services.IFilterService;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.designer.core.ICamelDesignerCoreService;
 import org.talend.designer.maven.launch.MavenPomCommandLauncher;
+import org.talend.designer.maven.model.MavenSystemFolders;
 import org.talend.designer.maven.model.TalendJavaProjectConstants;
 import org.talend.designer.maven.model.TalendMavenConstants;
 import org.talend.designer.maven.template.MavenTemplateManager;
@@ -103,27 +115,25 @@ public class AggregatorPomsHelper {
         return projectTechName;
     }
 
-    public void createRootPom(List<String> modules, boolean force, IProgressMonitor monitor) throws Exception {
+    public void createRootPom(Model model, boolean force, IProgressMonitor monitor)
+            throws Exception {
         IFile pomFile = getProjectRootPom();
         if (force || !pomFile.exists()) {
-            Map<String, Object> parameters = new HashMap<String, Object>();
-            parameters.put(MavenTemplateManager.KEY_PROJECT_NAME, projectTechName);
-            Model model = MavenTemplateManager.getCodeProjectTemplateModel(parameters);
-            if (modules != null && !modules.isEmpty()) {
-                model.setModules(modules);
-            }
             PomUtil.savePom(monitor, model, pomFile);
         }
     }
 
     public void createRootPom(IProgressMonitor monitor) throws Exception {
+        Model newModel = getCodeProjectTemplateModel();
         IFile pomFile = getProjectRootPom();
-        List<String> modules = null;
         if (pomFile != null && pomFile.exists()) {
-            Model model = MavenPlugin.getMavenModelManager().readMavenModel(pomFile);
-            modules = model.getModules();
+            Model oldModel = MavenPlugin.getMavenModelManager().readMavenModel(pomFile);
+            List<Profile> profiles = oldModel.getProfiles().stream()
+                    .filter(profile -> matchModuleProfile(profile.getId(), projectTechName)).collect(Collectors.toList());
+            newModel.setModules(oldModel.getModules());
+            newModel.getProfiles().addAll(profiles);
         }
-        createRootPom(modules, true, monitor);
+        createRootPom(newModel, true, monitor);
     }
 
     public void installRootPom(boolean force) throws Exception {
@@ -182,7 +192,14 @@ public class AggregatorPomsHelper {
     private void updateCodeProject(IProgressMonitor monitor, ERepositoryObjectType codeType, boolean forceBuild) {
         try {
             ITalendProcessJavaProject codeProject = getCodesProject(codeType);
-            updateCodeProjectPom(monitor, codeType, codeProject.getProjectPom());
+            IFile pomFile;
+            if (codeProject == null) {
+                // try to recover from any damage of pom.
+                pomFile = getCodeFolder(codeType).getFile(TalendMavenConstants.POM_FILE_NAME);
+            } else {
+                pomFile = codeProject.getProjectPom();
+            }
+            updateCodeProjectPom(monitor, codeType, pomFile);
             buildAndInstallCodesProject(monitor, codeType, true, forceBuild);
         } catch (Exception e) {
             ExceptionHandler.process(e);
@@ -283,37 +300,40 @@ public class AggregatorPomsHelper {
         }
     }
 
-    public void updateRefProjectModules(List<ProjectReference> references) {
+    public void updateRefProjectModules(List<ProjectReference> references, IProgressMonitor monitor) {
         if (!needUpdateRefProjectModules()) {
             return;
         }
         try {
-            List<String> modules = new ArrayList<>();
-            for (ProjectReference reference : references) {
-                String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
-                String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
-                modules.add(modulePath);
-            }
-
-            Project mainProject = ProjectManager.getInstance().getCurrentProject();
-            IFolder mainPomsFolder = new AggregatorPomsHelper(mainProject.getTechnicalLabel()).getProjectPomsFolder();
-            IFile mainPomFile = mainPomsFolder.getFile(TalendMavenConstants.POM_FILE_NAME);
-
-            Model model = MavenPlugin.getMavenModelManager().readMavenModel(mainPomFile);
-            List<String> oldModules = model.getModules();
-            if (oldModules == null) {
-                oldModules = new ArrayList<>();
-            }
-            ListIterator<String> iterator = oldModules.listIterator();
-            while (iterator.hasNext()) {
-                String modulePath = iterator.next();
-                if (modulePath.startsWith("../../")) { //$NON-NLS-1$
-                    iterator.remove();
+            Model model = MavenPlugin.getMavenModelManager().readMavenModel(getProjectRootPom());
+            if (PomIdsHelper.useProfileModule()) {
+                List<Profile> profiles = collectRefProjectProfiles(references);
+                Iterator<Profile> iterator = model.getProfiles().listIterator();
+                while (iterator.hasNext()) {
+                    Profile profile = iterator.next();
+                    if (matchModuleProfile(profile.getId(), projectTechName)) {
+                        iterator.remove();
+                    }
                 }
+                model.getProfiles().addAll(profiles);
+            } else {
+                List<String> refPrjectModules = new ArrayList<>();
+                references.forEach(reference -> {
+                    String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
+                    String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
+                    refPrjectModules.add(modulePath);
+                });
+                List<String> modules = model.getModules();
+                Iterator<String> iterator = modules.listIterator();
+                while (iterator.hasNext()) {
+                    String modulePath = iterator.next();
+                    if (modulePath.startsWith("../../")) { //$NON-NLS-1$
+                        iterator.remove();
+                    }
+                }
+                modules.addAll(refPrjectModules);
             }
-            oldModules.addAll(modules);
-
-            PomUtil.savePom(null, model, mainPomFile);
+            createRootPom(model, true, monitor);
         } catch (Exception e) {
             ExceptionHandler.process(e);
         }
@@ -487,6 +507,10 @@ public class AggregatorPomsHelper {
         return null;
     }
 
+    public IFolder getCodeSrcFolder(ERepositoryObjectType codeType) {
+        return getCodeFolder(codeType).getFolder(MavenSystemFolders.JAVA.getPath());
+    }
+
     public IFolder getProcessFolder(ERepositoryObjectType type) {
         return getProjectPomsFolder().getFolder(DIR_JOBS).getFolder(type.getFolder());
     }
@@ -517,9 +541,16 @@ public class AggregatorPomsHelper {
     }
 
     /**
-     * without create/open project
+     * without create/open project<br/>
+     * Use Function to get the relativePath from property at realtime, since the property may be changed
      */
-    public static IFolder getItemPomFolder(Property property, String realVersion) {
+    public static IFolder getItemPomFolder(Property property, String realVersion, Function<Property, IPath> getItemRelativePath) {
+        return getItemPomFolder(property, ProjectManager.getInstance().getProject(property).getTechnicalLabel(), realVersion,
+                getItemRelativePath);
+    }
+
+    public static IFolder getItemPomFolder(Property property, String projectTechName, String realVersion,
+            Function<Property, IPath> getItemRelativePath) {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
             ITestContainerProviderService testContainerService =
                     (ITestContainerProviderService) GlobalServiceRegister.getDefault().getService(
@@ -535,17 +566,25 @@ public class AggregatorPomsHelper {
                 }
             }
         }
-        String projectTechName = ProjectManager.getInstance().getProject(property).getTechnicalLabel();
         AggregatorPomsHelper helper = new AggregatorPomsHelper(projectTechName);
-        IPath itemRelativePath = ItemResourceUtil.getItemRelativePath(property);
+        IPath itemRelativePath = getItemRelativePath.apply(property);
         String version = realVersion == null ? property.getVersion() : realVersion;
         String jobFolderName = getJobProjectFolderName(property.getLabel(), version);
         ERepositoryObjectType type = ERepositoryObjectType.getItemType(property.getItem());
-        IFolder jobFolder = helper.getProcessFolder(type).getFolder(itemRelativePath).getFolder(jobFolderName);
+        IFolder jobFolder = null;
+        if (PomIdsHelper.skipFolders()) {
+            jobFolder = helper.getProcessFolder(type).getFolder(jobFolderName);
+        } else {
+            jobFolder = helper.getProcessFolder(type).getFolder(itemRelativePath).getFolder(jobFolderName);
+        }
         createFoldersIfNeeded(jobFolder);
         return jobFolder;
     }
 
+    public static IFolder getItemPomFolder(Property property, String realVersion) {
+        return getItemPomFolder(property, realVersion, p -> ItemResourceUtil.getItemRelativePath(p));
+    }
+
     private static void createFoldersIfNeeded(IFolder folder) {
         if (!folder.exists()) {
             if (folder.getParent() instanceof IFolder) {
@@ -671,6 +710,23 @@ public class AggregatorPomsHelper {
         new ProgressMonitorDialog(Display.getDefault().getActiveShell()).run(true, true, runnableWithProgress);
     }
 
+    public void syncParentJobPomsForPropertyChange(Property property) {
+        IRunProcessService runProcessService = getRunProcessService();
+        ProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
+        List<Relation> itemsHaveRelationWith = RelationshipItemBuilder.getInstance().getItemsHaveRelationWith(property.getId(),
+                property.getVersion());
+        try {
+            for (Relation relation : itemsHaveRelationWith) {
+                IRepositoryViewObject object = factory.getSpecificVersion(relation.getId(), relation.getVersion(), true);
+                if (runProcessService != null) {
+                    runProcessService.generatePom(object.getProperty().getItem());
+                }
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
     private String getModulePath(IFile pomFile) {
         IFile parentPom = getProjectRootPom();
         if (parentPom != null) {
@@ -680,47 +736,48 @@ public class AggregatorPomsHelper {
         return null;
     }
 
-    private void collectModules(List<String> modules) {
-        IRunProcessService service = getRunProcessService();
-        if (service != null) {
-            modules.add(
-                    getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.ROUTINES).getProjectPom()));
-            if (ProcessUtils.isRequiredPigUDFs(null)) {
-                modules.add(
-                        getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.PIG_UDF).getProjectPom()));
-            }
-            if (ProcessUtils.isRequiredBeans(null)) {
-                modules.add(getModulePath(service
-                        .getTalendCodeJavaProject(ERepositoryObjectType.valueOf("BEANS")) //$NON-NLS-1$
-                        .getProjectPom()));
-            }
+    private List<Profile> collectRefProjectProfiles(List<ProjectReference> references) throws CoreException {
+        if (!needUpdateRefProjectModules()) {
+            Model model = MavenPlugin.getMavenModelManager().readMavenModel(getProjectRootPom());
+            List<Profile> profiles = model.getProfiles();
+            return profiles.stream().filter(profile -> matchModuleProfile(profile.getId(), projectTechName))
+                    .collect(Collectors.toList());
         }
-        if (needUpdateRefProjectModules()) {
-            List<ProjectReference> references =
-                    ProjectManager.getInstance().getCurrentProject().getProjectReferenceList(true);
-            for (ProjectReference reference : references) {
-                String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
-                String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
-                modules.add(modulePath);
-            }
-        } else {
-            Project mainProject = ProjectManager.getInstance().getCurrentProject();
-            IFolder mainPomsFolder = new AggregatorPomsHelper(mainProject.getTechnicalLabel()).getProjectPomsFolder();
-            IFile mainPomFile = mainPomsFolder.getFile(TalendMavenConstants.POM_FILE_NAME);
-            try {
-                Model model = MavenPlugin.getMavenModelManager().readMavenModel(mainPomFile);
-                List<String> oldModules = model.getModules();
-                if (oldModules != null) {
-                    for (String modulePath : oldModules) {
-                        if (modulePath.startsWith("../../")) { //$NON-NLS-1$
-                            modules.add(modulePath);
-                        }
-                    }
-                }
-            } catch (CoreException e) {
-                ExceptionHandler.process(e);
-            }
+        if (references == null) {
+            references = ProjectManager.getInstance().getCurrentProject().getProjectReferenceList(true);
         }
+        List<Profile> profiles = new ArrayList<>();
+        references.forEach(reference -> {
+            String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
+            String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
+            Profile profile = new Profile();
+            profile.setId((projectTechName + "_" + refProjectTechName).toLowerCase()); //$NON-NLS-1$
+            Activation activation = new Activation();
+            activation.setActiveByDefault(true);
+            profile.setActivation(activation);
+            profile.getModules().add(modulePath);
+            profiles.add(profile);
+        });
+        return profiles;
+    }
+
+    private List<String> collectRefProjectModules(List<ProjectReference> references) throws CoreException {
+        if (!needUpdateRefProjectModules()) {
+            Model model = MavenPlugin.getMavenModelManager().readMavenModel(getProjectRootPom());
+            return model.getModules().stream().filter(modulePath -> modulePath.startsWith("../../")) //$NON-NLS-1$
+                    .collect(Collectors.toList());
+        }
+        if (references == null) {
+            references = ProjectManager.getInstance().getCurrentProject().getProjectReferenceList(true);
+        }
+        List<String> modules = new ArrayList<>();
+        references.forEach(reference -> {
+            String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
+            String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
+            modules.add(modulePath);
+        });
+        return modules;
+
     }
 
     public boolean needUpdateRefProjectModules() {
@@ -758,7 +815,13 @@ public class AggregatorPomsHelper {
         monitor.beginTask("", size); //$NON-NLS-1$
         // project pom
         monitor.subTask("Synchronize project pom"); //$NON-NLS-1$
-        createRootPom(null, true, monitor);
+        Model model = getCodeProjectTemplateModel();
+        if (PomIdsHelper.useProfileModule()) {
+            model.getProfiles().addAll(collectRefProjectProfiles(null));
+        } else {
+            model.getModules().addAll(collectRefProjectModules(null));
+        }
+        createRootPom(model, true, monitor);
         installRootPom(true);
         monitor.worked(1);
         if (monitor.isCanceled()) {
@@ -811,8 +874,9 @@ public class AggregatorPomsHelper {
         }
         // sync project pom again with all modules.
         monitor.subTask("Synchronize project pom with modules"); //$NON-NLS-1$
-        collectModules(modules);
-        createRootPom(modules, true, monitor);
+        collectCodeModules(modules);
+        model.getModules().addAll(modules);
+        createRootPom(model, true, monitor);
         installRootPom(true);
         monitor.worked(1);
         if (monitor.isCanceled()) {
@@ -821,6 +885,21 @@ public class AggregatorPomsHelper {
         monitor.done();
     }
 
+    private void collectCodeModules(List<String> modules) {
+        // collect codes modules
+        IRunProcessService service = getRunProcessService();
+        if (service != null) {
+            modules.add(getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.ROUTINES).getProjectPom()));
+            if (ProcessUtils.isRequiredPigUDFs(null)) {
+                modules.add(getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.PIG_UDF).getProjectPom()));
+            }
+            if (ProcessUtils.isRequiredBeans(null)) {
+                modules.add(getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.valueOf("BEANS")) //$NON-NLS-1$
+                        .getProjectPom()));
+            }
+        }
+    }
+
     /**
      * Check if is a esb data service job
      * 
@@ -849,6 +928,18 @@ public class AggregatorPomsHelper {
         return null;
     }
 
+    private Model getCodeProjectTemplateModel() {
+        Map<String, Object> parameters = new HashMap<String, Object>();
+        parameters.put(MavenTemplateManager.KEY_PROJECT_NAME, projectTechName);
+        return MavenTemplateManager.getCodeProjectTemplateModel(parameters);
+    }
+
+    public static boolean matchModuleProfile(String profileId, String projectTechName) {
+        // FIXME get profile id from extension point.
+        List<String> otherProfiles = Arrays.asList("docker", "cloud-publisher", "nexus"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+        return !otherProfiles.contains(profileId) && StringUtils.startsWithIgnoreCase(profileId, projectTechName + "_");
+    }
+
     private static IRunProcessService getRunProcessService() {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
             IRunProcessService runProcessService =

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/BuildCacheManager.java

@@ -24,7 +24,6 @@ import java.util.Set;
 
 import org.apache.maven.model.Model;
 import org.eclipse.core.resources.IFile;
-import org.eclipse.core.resources.ResourcesPlugin;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
@@ -47,7 +46,6 @@ import org.talend.designer.maven.utils.MavenProjectUtils;
 import org.talend.designer.maven.utils.PomUtil;
 import org.talend.designer.runprocess.IRunProcessService;
 import org.talend.repository.ProjectManager;
-import org.talend.repository.model.IRepositoryService;
 
 /**
  * DOC zwxue class global comment. Detailled comment
@@ -271,6 +269,7 @@ public class BuildCacheManager {
 
     public void clearAllCaches() {
         jobCache.clear();
+        clearCurrentJobletCache();
         jobletCache.clear();
         codesLastBuildCache.clear();
     }
@@ -330,38 +329,9 @@ public class BuildCacheManager {
     }
 
     private String getModulePath(Property property) {
-        String modulePath = null;
-        IPath basePath = null;
         IPath jobProjectPath = AggregatorPomsHelper.getItemPomFolder(property).getLocation();
-        ProjectManager proManager = ProjectManager.getInstance();
-        if (!proManager.isInCurrentMainProject(property)) {
-            if (GlobalServiceRegister.getDefault().isServiceRegistered(IRepositoryService.class)) {
-                IRepositoryService service = (IRepositoryService) GlobalServiceRegister.getDefault()
-                        .getService(IRepositoryService.class);
-                if (service.isGIT()) {
-                    String refProjectTechName = proManager.getCurrentProject().getTechnicalLabel();
-                    String mainProjectBranch = proManager.getMainProjectBranch(refProjectTechName);
-                    if ("master".equals(mainProjectBranch)) {
-                        modulePath = "../../../../"; //$NON-NLS-1$
-                    } else {
-                        modulePath = "../../../../../"; //$NON-NLS-1$
-                    }
-                    basePath = ResourcesPlugin.getWorkspace().getRoot().getLocation().append("/.repositories"); //$NON-NLS-1$
-                } else if (service.isSVN()) {
-                    modulePath = "../../"; //$NON-NLS-1$
-                    basePath = ResourcesPlugin.getWorkspace().getRoot().getLocation();
-                }
-            }
-            if (modulePath == null || basePath == null) {
-                throw new RuntimeException("modulePath or basePath can not be null!"); //$NON-NLS-1$
-            }
-        } else {
-            modulePath = ""; //$NON-NLS-1$
-            basePath = getAggregatorPomsHelper().getProjectPomsFolder().getLocation();
-        }
-        jobProjectPath = jobProjectPath.makeRelativeTo(basePath);
-        modulePath += jobProjectPath.toPortableString();
-
+        IPath basePath = getAggregatorPomsHelper().getProjectPomsFolder().getLocation();
+        String modulePath = jobProjectPath.makeRelativeTo(basePath).toPortableString();
         return modulePath;
     }
 

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/ProjectPomManager.java

@@ -69,13 +69,7 @@ public class ProjectPomManager {
         if (monitor == null) {
             monitor = new NullProgressMonitor();
         }
-        Model projectModel = MODEL_MANAGER.readMavenModel(projectPomFile);
-        Model templateModel = MavenTemplateManager.getCodeProjectTemplateModel();
-        for (String module : projectModel.getModules()) {
-            templateModel.addModule(module);
-        }
- 
-        PomUtil.savePom(monitor, templateModel, projectPomFile);
+        new AggregatorPomsHelper().createRootPom(monitor);
     }
 
     /**

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/AbstractMavenProcessorPom.java

@@ -45,12 +45,14 @@ import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenConstants;
 import org.talend.core.runtime.maven.MavenUrlHelper;
+import org.talend.core.runtime.process.LastGenerationInfo;
 import org.talend.core.runtime.projectsetting.IProjectSettingTemplateConstants;
 import org.talend.core.runtime.repository.build.IMavenPomCreator;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.designer.maven.model.TalendMavenConstants;
 import org.talend.designer.maven.template.ETalendMavenVariables;
 import org.talend.designer.maven.tools.ProcessorDependenciesManager;
+import org.talend.designer.maven.utils.JobUtils;
 import org.talend.designer.maven.utils.PomIdsHelper;
 import org.talend.designer.maven.utils.PomUtil;
 import org.talend.designer.runprocess.IBigDataProcessor;
@@ -133,9 +135,15 @@ public abstract class AbstractMavenProcessorPom extends CreateMavenBundleTemplat
 
         Map<ETalendMavenVariables, String> variablesValuesMap = new HashMap<ETalendMavenVariables, String>();
         // no need check property is null or not, because if null, will get default ids.
-        variablesValuesMap.put(ETalendMavenVariables.JobGroupId, PomIdsHelper.getJobGroupId(property));
+        JobInfo lastMainJob = LastGenerationInfo.getInstance().getLastMainJob(); 
+        if (JobUtils.isJob(property) && ProcessUtils.isChildRouteProcess(process) && lastMainJob != null) {
+            variablesValuesMap.put(ETalendMavenVariables.JobGroupId, PomIdsHelper.getJobGroupId(lastMainJob.getProcessor().getProperty()));
+            variablesValuesMap.put(ETalendMavenVariables.JobVersion, PomIdsHelper.getJobVersion(lastMainJob.getProcessor().getProperty()));	    	
+        }else {
+            variablesValuesMap.put(ETalendMavenVariables.JobGroupId, PomIdsHelper.getJobGroupId(property));
+            variablesValuesMap.put(ETalendMavenVariables.JobVersion, PomIdsHelper.getJobVersion(property));	    	
+        }
         variablesValuesMap.put(ETalendMavenVariables.JobArtifactId, PomIdsHelper.getJobArtifactId(property));
-        variablesValuesMap.put(ETalendMavenVariables.JobVersion, PomIdsHelper.getJobVersion(property));
         variablesValuesMap.put(ETalendMavenVariables.TalendJobVersion, property.getVersion());
         final String jobName = JavaResourcesHelper.escapeFileName(process.getName());
         variablesValuesMap.put(ETalendMavenVariables.JobName, jobName);
@@ -208,11 +216,12 @@ public abstract class AbstractMavenProcessorPom extends CreateMavenBundleTemplat
                 Xpp3Dom artifactSet = new Xpp3Dom("artifactSet"); //$NON-NLS-1$
                 configuration.addChild(artifactSet);
                 Xpp3Dom excludes = new Xpp3Dom("excludes"); //$NON-NLS-1$
-                if (!bigDataProcessor.getShadedModulesExclude().isEmpty()) {
+                Set<ModuleNeeded> modules = bigDataProcessor.getShadedModulesExclude();
+                if (!modules.isEmpty()) {
                     artifactSet.addChild(excludes);
                 }
 
-                for (ModuleNeeded module : bigDataProcessor.getShadedModulesExclude()) {
+                for (ModuleNeeded module : modules) {
                     Xpp3Dom include = new Xpp3Dom("exclude"); //$NON-NLS-1$
                     excludes.addChild(include);
                     MavenArtifact mvnArtifact = MavenUrlHelper.parseMvnUrl(module.getMavenUri());
@@ -281,9 +290,15 @@ public abstract class AbstractMavenProcessorPom extends CreateMavenBundleTemplat
                 String type = null;
                 if (!jobInfo.isJoblet()) {
                     property = jobInfo.getProcessItem().getProperty();
-                    groupId = PomIdsHelper.getJobGroupId(property);
                     artifactId = PomIdsHelper.getJobArtifactId(jobInfo);
-                    version = PomIdsHelper.getJobVersion(property);
+                    JobInfo lastMainJob = LastGenerationInfo.getInstance().getLastMainJob();
+                    if (lastMainJob != null && JobUtils.isJob(jobInfo) && JobUtils.isRoute(getJobProcessor().getProperty())) {
+                        groupId = PomIdsHelper.getJobGroupId(lastMainJob.getProcessor().getProperty());
+                        version =  PomIdsHelper.getJobVersion(lastMainJob.getProcessor().getProperty()); 
+                    } else {
+                        groupId = PomIdsHelper.getJobGroupId(property);	
+                        version = PomIdsHelper.getJobVersion(property);
+                    }
                     // try to get the pom version of children job and load from the pom file.
                     String childPomFileName = PomUtil.getPomFileName(jobInfo.getJobName(), jobInfo.getJobVersion());
                     IProject codeProject = getJobProcessor().getCodeProject();

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenJobPom.java

@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
+import java.util.Scanner;
 import java.util.Set;
 
 import org.apache.commons.lang3.BooleanUtils;
@@ -55,6 +56,7 @@ import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Project;
 import org.talend.core.model.properties.Property;
+import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.repository.utils.ItemResourceUtil;
 import org.talend.core.runtime.maven.MavenArtifact;
@@ -75,6 +77,7 @@ import org.talend.designer.core.model.utils.emf.talendfile.ContextParameterType;
 import org.talend.designer.maven.model.TalendMavenConstants;
 import org.talend.designer.maven.template.ETalendMavenVariables;
 import org.talend.designer.maven.template.MavenTemplateManager;
+import org.talend.designer.maven.utils.JobUtils;
 import org.talend.designer.maven.utils.PomIdsHelper;
 import org.talend.designer.maven.utils.PomUtil;
 import org.talend.designer.runprocess.IProcessor;
@@ -203,15 +206,12 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
             }
         }
 
-        // same as JavaProcessor.initCodePath
-        String jobClassPackageFolder = JavaResourcesHelper.getJobClassPackageFolder(property.getItem());
-
         Project project = ProjectManager.getInstance().getProject(property);
         if (project == null) { // current project
             project = ProjectManager.getInstance().getCurrentProject().getEmfProject();
         }
-
-        checkPomProperty(properties, "talend.job.path", ETalendMavenVariables.JobPath, jobClassPackageFolder);
+        String talendJobPath = project.getTechnicalLabel().toLowerCase();
+        checkPomProperty(properties, "talend.job.path", ETalendMavenVariables.JobPath, talendJobPath);
         IPath jobFolderPath = ItemResourceUtil.getItemRelativePath(property);
         String jobFolder = "";
         if (jobFolderPath != null && !StringUtils.isEmpty(jobFolderPath.toPortableString())) {
@@ -561,6 +561,7 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
                         "${talend.job.bat.addition}" },
                 new String[] { jvmArgsStr.toString().trim(), getWindowsClasspath(), jobClass,
                         windowsScriptAdditionValue.toString() });
+        batContent = normalizeSpaces(batContent);
 
         String shContent = MavenTemplateManager.getProjectSettingValue(IProjectSettingPreferenceConstants.TEMPLATE_SH,
                 templateParameters);
@@ -569,6 +570,7 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
                         "${talend.job.sh.addition}" },
                 new String[] { jvmArgsStr.toString().trim(), getUnixClasspath(), jobClass,
                         unixScriptAdditionValue.toString() });
+        shContent = normalizeSpaces(shContent);
 
         String psContent = MavenTemplateManager.getProjectSettingValue(IProjectSettingPreferenceConstants.TEMPLATE_PS,
                 templateParameters);
@@ -577,6 +579,7 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
                         "${talend.job.bat.addition}" },
                 new String[] { jvmArgsStrPs1.toString().trim(), getWindowsClasspathForPs1(), jobClass,
                         windowsScriptAdditionValue.toString() });
+        psContent = normalizeSpaces(psContent);
 
         String jobInfoContent = MavenTemplateManager
                 .getProjectSettingValue(IProjectSettingPreferenceConstants.TEMPLATE_JOB_INFO, templateParameters);
@@ -595,7 +598,10 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
                 mainProjectBranch = "";
             }
         }
-        jobInfoContent = StringUtils.replace(jobInfoContent, "${talend.project.branch}", mainProjectBranch);
+
+        if (!isOptionChecked(TalendProcessArgumentConstant.ARG_AVOID_BRANCH_NAME)) {
+            jobInfoContent = StringUtils.replace(jobInfoContent, "${talend.project.branch}", mainProjectBranch);
+        }
 
         IFolder templateFolder = codeProject.getTemplatesFolder();
         IFile shFile = templateFolder.getFile(IProjectSettingTemplateConstants.JOB_RUN_SH_TEMPLATE_FILE_NAME);
@@ -608,31 +614,66 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         MavenTemplateManager.saveContent(psFile, psContent, overwrite);
         MavenTemplateManager.saveContent(infoFile, jobInfoContent, overwrite);
     }
+    
+    // https://jira.talendforge.org/browse/TUP-27053
+    // https://jira.talendforge.org/browse/TUP-24422(Also fix the carriage return issue in Windows)
+    public static String normalizeSpaces(String src) {
+        StringBuffer sb = new StringBuffer();
+        try (Scanner scanner = new Scanner(src)) {
+            while (scanner.hasNextLine()) {
+                String line = scanner.nextLine();
+                line = StringUtils.normalizeSpace(line.trim());
+                if (!line.isEmpty()) {
+                    sb.append(line);
+                }
+                sb.append('\n');
+            }
+        } catch (Exception e) {
+
+        }
+        return sb.toString();
+    }
 
     protected void updateDependencySet(IFile assemblyFile) {
-        Set<String> jobCoordinate = new HashSet<>();
+        Map<String, Dependency> jobCoordinateMap = new HashMap<String, Dependency>();
         Set<JobInfo> childrenJobInfo = new HashSet<>();
+        childrenJobInfo = getJobProcessor().getBuildChildrenJobs();
         if (!hasLoopDependency()) {
-            childrenJobInfo = getJobProcessor().getBuildChildrenJobs();
+            // add children jobs
+            for (JobInfo jobInfo : childrenJobInfo) {
+                Property property = jobInfo.getProcessItem().getProperty();
+                String coordinate = getCoordinate(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo),
+                        MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(property), null);
+                String groupId = null;
+                String version = null;
+                IProcessor jobProcessor = getJobProcessor();
+                 if (JobUtils.isJob(jobInfo) && jobProcessor !=null &&  JobUtils.isRoute(getJobProcessor().getProperty()))  {
+                    groupId = PomIdsHelper.getJobGroupId(jobProcessor.getProperty());     
+                    version = PomIdsHelper.getJobVersion(jobProcessor.getProperty());
+                }else {
+                    groupId = PomIdsHelper.getJobGroupId(property);     
+                    version = PomIdsHelper.getJobVersion(property);                    	
+                }
+                Dependency dependency = PomUtil.createDependency(groupId,
+                        PomIdsHelper.getJobArtifactId(jobInfo), version,
+                        MavenConstants.PACKAGING_JAR);
+                jobCoordinateMap.put(coordinate, dependency);
+            }
         }
 
-        // add children jobs
-        for (JobInfo jobInfo : childrenJobInfo) {
-            Property property = jobInfo.getProcessItem().getProperty();
-            String coordinate = getCoordinate(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo),
-                    MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(property));
-            jobCoordinate.add(coordinate);
-        }
 
         // add parent job
         Property parentProperty = this.getJobProcessor().getProperty();
-        String parentCoordinate = getCoordinate(PomIdsHelper.getJobGroupId(parentProperty),
-                PomIdsHelper.getJobArtifactId(parentProperty), MavenConstants.PACKAGING_JAR,
-                PomIdsHelper.getJobVersion(parentProperty));
-        jobCoordinate.add(parentCoordinate);
+        String parentCoordinate =
+                getCoordinate(PomIdsHelper.getJobGroupId(parentProperty), PomIdsHelper.getJobArtifactId(parentProperty),
+                        MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(parentProperty), null);
+        Dependency parentDependency = PomUtil.createDependency(PomIdsHelper.getJobGroupId(parentProperty),
+                PomIdsHelper.getJobArtifactId(parentProperty), PomIdsHelper.getJobVersion(parentProperty),
+                MavenConstants.PACKAGING_JAR);
+        jobCoordinateMap.put(parentCoordinate, parentDependency);
 
         // add talend libraries and codes
-        Set<String> talendLibCoordinate = new HashSet<>();
+        Map<String, Dependency> talendLibCoordinateMap = new HashMap<String, Dependency>();
         String projectTechName = ProjectManager.getInstance().getProject(parentProperty).getTechnicalLabel();
         String projectGroupId = PomIdsHelper.getProjectGroupId(projectTechName);
 
@@ -640,13 +681,14 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         List<Dependency> dependencies = new ArrayList<>();
         addCodesDependencies(dependencies);
         for (Dependency dependency : dependencies) {
-            talendLibCoordinate.add(getCoordinate(dependency));
+            talendLibCoordinateMap.put(getCoordinate(dependency), dependency);
         }
 
         // libraries
         dependencies.clear();
-        Set<ModuleNeeded> modules = getJobProcessor().getNeededModules(
-                TalendProcessOptionConstants.MODULES_WITH_JOBLET);
+        Set<ModuleNeeded> modules =
+                getJobProcessor().getNeededModules(
+                        TalendProcessOptionConstants.MODULES_WITH_JOBLET | TalendProcessOptionConstants.MODULES_EXCLUDE_SHADED);
         for (ModuleNeeded module : modules) {
             String mavenUri = module.getMavenUri();
             Dependency dependency = PomUtil.createModuleDependency(mavenUri);
@@ -658,23 +700,23 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
             }
             String dependencyGroupId = dependency.getGroupId();
             String coordinate = getCoordinate(dependency);
-            if (!jobCoordinate.contains(coordinate)) {
+            if (!jobCoordinateMap.containsKey(coordinate)) {
                 if (MavenConstants.DEFAULT_LIB_GROUP_ID.equals(dependencyGroupId)) {
-                    talendLibCoordinate.add(coordinate);
+                    talendLibCoordinateMap.put(coordinate, dependency);
                 }
             }
         }
 
         // add 3rd party libraries
-        Set<String> _3rdDepLib = new HashSet<>();
+        Map<String, Dependency> _3rdDepLibMap = new HashMap<String, Dependency>();
         Map<String, Set<Dependency>> duplicateLibs = new HashMap<>();
         for (Dependency dependency : dependencies) {
             if (MavenConstants.PACKAGING_POM.equals(dependency.getType())) {
                 continue;
             }
             String coordinate = getCoordinate(dependency);
-            if (!jobCoordinate.contains(coordinate) && !talendLibCoordinate.contains(coordinate)) {
-                _3rdDepLib.add(coordinate);
+            if (!jobCoordinateMap.containsKey(coordinate) && !talendLibCoordinateMap.containsKey(coordinate)) {
+                _3rdDepLibMap.put(coordinate, dependency);
                 addToDuplicateLibs(duplicateLibs, dependency);
             }
         }
@@ -686,22 +728,45 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
                     jobInfo.getJobVersion()));
         }
         for (ModuleNeeded moduleNeeded : fullModulesList) {
+            if (moduleNeeded.isExcluded()) {
+                continue;
+            }
             MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(moduleNeeded.getMavenUri());
             String coordinate = getCoordinate(artifact.getGroupId(), artifact.getArtifactId(), artifact.getType(),
-                    artifact.getVersion());
-            if (!jobCoordinate.contains(coordinate) && !talendLibCoordinate.contains(coordinate)
-                    && !_3rdDepLib.contains(coordinate)) {
+                    artifact.getVersion(), artifact.getClassifier());
+            if (!jobCoordinateMap.containsKey(coordinate) && !talendLibCoordinateMap.containsKey(coordinate)
+                    && !_3rdDepLibMap.containsKey(coordinate)) {
+                Dependency dependencyObject = PomUtil.createDependency(artifact.getGroupId(), artifact.getArtifactId(),
+                        artifact.getVersion(), artifact.getType(), artifact.getClassifier());
                 if (MavenConstants.DEFAULT_LIB_GROUP_ID.equals(artifact.getGroupId())
                         || artifact.getGroupId().startsWith(projectGroupId)) {
-                    talendLibCoordinate.add(coordinate);
+                    talendLibCoordinateMap.put(coordinate, dependencyObject);
                 } else {
-                    _3rdDepLib.add(coordinate);
-                    Dependency dependency = PomUtil.createDependency(artifact.getGroupId(), artifact.getArtifactId(),
-                            artifact.getVersion(), artifact.getType(), artifact.getClassifier());
+                    _3rdDepLibMap.put(coordinate, dependencyObject);
+                    Dependency dependency = PomUtil
+                            .createDependency(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion(),
+                                    artifact.getType(), artifact.getClassifier());
                     addToDuplicateLibs(duplicateLibs, dependency);
                 }
             }
         }
+        
+        try {
+            Set<Dependency> codesDependencies = new HashSet<Dependency>();
+            for (ERepositoryObjectType type : ERepositoryObjectType.getAllTypesOfCodes()) {
+                codesDependencies.addAll(PomUtil.getCodesDependencies(type));
+            }
+            for (Dependency codeDependency : codesDependencies) {
+                Dependency dependency = PomUtil.createDependency(codeDependency.getGroupId(), codeDependency.getArtifactId(),
+                        codeDependency.getVersion(), codeDependency.getType(), codeDependency.getClassifier());
+                if (dependency != null) {
+                    addToDuplicateLibs(duplicateLibs, dependency);
+                }
+            }
+        } catch (CoreException e1) {
+            ExceptionHandler.process(e1);
+        }
+
 
         Iterator<String> iterator = duplicateLibs.keySet().iterator();
         while (iterator.hasNext()) {
@@ -713,7 +778,7 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
             } else {
                 // remove duplicated dependencies from 3rd lib list
                 for (Dependency dependency : dupDependencies) {
-                    _3rdDepLib.remove(getCoordinate(dependency));
+                    _3rdDepLibMap.remove(getCoordinate(dependency));
                 }
             }
         }
@@ -721,12 +786,13 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         try {
             Document document = PomUtil.loadAssemblyFile(null, assemblyFile);
             // add talend libs & codes
-            setupDependencySetNode(document, talendLibCoordinate, "lib", "${artifact.artifactId}.${artifact.extension}", false);
+            setupDependencySetNode(document, talendLibCoordinateMap, "lib", "${artifact.artifactId}.${artifact.extension}",
+                    false, false);
             // add 3rd party libs <dependencySet>
-            setupDependencySetNode(document, _3rdDepLib, "lib", null, false);
+            setupDependencySetNode(document, _3rdDepLibMap, "lib", null, false, false);
             // add jobs
-            setupDependencySetNode(document, jobCoordinate, "${talend.job.name}",
-                    "${artifact.build.finalName}.${artifact.extension}", true);
+            setupDependencySetNode(document, jobCoordinateMap, "${talend.job.name}",
+                    "${artifact.build.finalName}.${artifact.extension}", true, false);
             // add duplicate dependencies if exists
             setupFileNode(document, duplicateLibs);
 
@@ -737,24 +803,50 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
     }
 
     private String getCoordinate(Dependency dependency) {
-        return getCoordinate(dependency.getGroupId(), dependency.getArtifactId(), dependency.getType(), dependency.getVersion());
+        return getCoordinate(dependency.getGroupId(), dependency.getArtifactId(), dependency.getType(),
+                dependency.getVersion(), dependency.getClassifier());
     }
 
-    protected String getCoordinate(String groupId, String artifactId, String type, String version) {
+    protected String getCoordinate(String groupId, String artifactId, String type, String version, String classifier) {
         String separator = ":"; //$NON-NLS-1$
         String coordinate = groupId + separator;
         coordinate += artifactId + separator;
         if (type != null) {
             coordinate += type;
         }
+
         if (version != null) {
             coordinate += separator + version;
         }
+
+        if (StringUtils.isNotBlank(classifier)) {
+            coordinate += separator + classifier;
+        }
+
+        return coordinate;
+    }
+    
+    protected String getAssemblyCoordinate(Dependency dependency) {
+        String separator = ":"; //$NON-NLS-1$
+        String coordinate = dependency.getGroupId() + separator;
+        coordinate += dependency.getArtifactId() + separator;
+        if (dependency.getType() != null) {
+            coordinate += dependency.getType();
+        }
+        if (dependency.getClassifier() != null) {
+            coordinate += separator + "*";
+        }
+        if (dependency.getVersion() != null) {
+            coordinate += separator + dependency.getVersion();
+        }
+        
         return coordinate;
     }
 
     private void addToDuplicateLibs(Map<String, Set<Dependency>> map, Dependency dependency) {
-        String coordinate = getCoordinate(dependency.getGroupId(), dependency.getArtifactId(), dependency.getType(), null);
+        String coordinate =
+                getCoordinate(dependency.getGroupId(), dependency.getArtifactId(), dependency.getType(), null,
+                        dependency.getClassifier());
         if (!map.containsKey(coordinate)) {
             Set<Dependency> set = new HashSet<>();
             map.put(coordinate, set);
@@ -762,8 +854,8 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         map.get(coordinate).add(dependency);
     }
 
-    protected void setupDependencySetNode(Document document, Set<String> libIncludes, String outputDir, String fileNameMapping,
-            boolean useProjectArtifact) {
+    protected void setupDependencySetNode(Document document, Map<String, Dependency> libIncludes, String outputDir,
+            String fileNameMapping, boolean useProjectArtifact, boolean unpack) {
         if (libIncludes.isEmpty()) {
             return;
         }
@@ -774,16 +866,18 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         Node dependencySetNode = document.createElement("dependencySet");
         dependencySetsNode.appendChild(dependencySetNode);
 
-        Node outputDirNode = document.createElement("outputDirectory");
-        outputDirNode.setTextContent(outputDir);
-        dependencySetNode.appendChild(outputDirNode);
+        if (StringUtils.isNotBlank(outputDir)) {
+            Node outputDirNode = document.createElement("outputDirectory");
+            outputDirNode.setTextContent(outputDir);
+            dependencySetNode.appendChild(outputDirNode);
+        }
 
         Node includesNode = document.createElement("includes");
         dependencySetNode.appendChild(includesNode);
 
-        for (String include : libIncludes) {
+        for (Dependency dependency : libIncludes.values()) {
             Node includeNode = document.createElement("include");
-            includeNode.setTextContent(include);
+            includeNode.setTextContent(getAssemblyCoordinate(dependency));
             includesNode.appendChild(includeNode);
         }
 
@@ -797,6 +891,12 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         useProjectArtifactNode.setTextContent(Boolean.toString(useProjectArtifact));
         dependencySetNode.appendChild(useProjectArtifactNode);
 
+        if (unpack) {
+            Node unpackNode = document.createElement("unpack");
+            unpackNode.setTextContent(Boolean.TRUE.toString());
+            dependencySetNode.appendChild(unpackNode);
+        }
+
     }
 
     private void setupFileNode(Document document, Map<String, Set<Dependency>> duplicateDependencies) {

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenJobletPom.java

@@ -123,7 +123,9 @@ public class CreateMavenJobletPom extends AbstractMavenProcessorPom {
     @Override
     protected void addDependencies(Model model) {
         try {
-            getProcessorDependenciesManager().updateDependencies(null, model);
+            if (ERepositoryObjectType.JOBLET == ERepositoryObjectType.getType(getJobProcessor().getProperty())) {
+                getProcessorDependenciesManager().updateDependencies(null, model);
+            }
             addChildrenDependencies(model.getDependencies());
         } catch (ProcessorException e) {
             ExceptionHandler.process(e);

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenStandardJobOSGiPom.java

@@ -18,12 +18,12 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import org.apache.maven.model.Build;
+import org.apache.maven.model.Dependency;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.Plugin;
 import org.apache.maven.model.PluginExecution;
@@ -213,28 +213,34 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
 
     @Override
     protected void updateDependencySet(IFile assemblyFile) {
-        Set<String> jobCoordinate = new HashSet<>();
+        Map<String, Dependency> jobCoordinateMap = new HashMap<String, Dependency>();
         if (!hasLoopDependency()) {
             // add children jobs
             Set<JobInfo> childrenJobInfo = getJobProcessor().getBuildChildrenJobs();
             for (JobInfo jobInfo : childrenJobInfo) {
                 Property property = jobInfo.getProcessItem().getProperty();
                 String coordinate = getCoordinate(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo),
-                        MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(property));
-                jobCoordinate.add(coordinate);
+                        MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(property), null);
+                Dependency dependency = PomUtil.createDependency(PomIdsHelper.getJobGroupId(property),
+                        PomIdsHelper.getJobArtifactId(jobInfo), PomIdsHelper.getJobVersion(property),
+                        MavenConstants.PACKAGING_JAR);
+                jobCoordinateMap.put(coordinate, dependency);
             }
         }
         // add parent job
         Property parentProperty = this.getJobProcessor().getProperty();
         String parentCoordinate = getCoordinate(PomIdsHelper.getJobGroupId(parentProperty),
                 PomIdsHelper.getJobArtifactId(parentProperty), MavenConstants.PACKAGING_JAR,
-                PomIdsHelper.getJobVersion(parentProperty));
-        jobCoordinate.add(parentCoordinate);
+                PomIdsHelper.getJobVersion(parentProperty), null);
+        Dependency parentDependency = PomUtil.createDependency(PomIdsHelper.getJobGroupId(parentProperty),
+                PomIdsHelper.getJobArtifactId(parentProperty), PomIdsHelper.getJobVersion(parentProperty),
+                MavenConstants.PACKAGING_JAR);
+        jobCoordinateMap.put(parentCoordinate, parentDependency);
         try {
             Document document = PomUtil.loadAssemblyFile(null, assemblyFile);
             // add jobs
-            setupDependencySetNode(document, jobCoordinate, "${talend.job.name}",
-                    "${artifact.build.finalName}.${artifact.extension}", true);
+            setupDependencySetNode(document, jobCoordinateMap, null, "${artifact.build.finalName}.${artifact.extension}", true,
+                    true);
             PomUtil.saveAssemblyFile(assemblyFile, document);
         } catch (Exception e) {
             ExceptionHandler.process(e);

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/JobUtils.java

@@ -24,6 +24,8 @@ import org.talend.core.model.process.JobInfo;
 import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.ProcessItem;
+import org.talend.core.model.properties.Property;
+import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.runtime.CoreRuntimePlugin;
@@ -98,4 +100,28 @@ public class JobUtils {
         return clonedJobInfos;
     }
 
+    public static boolean isRoute(Property p) {
+        if (p != null) {
+            return ERepositoryObjectType.getType(p).equals(ERepositoryObjectType.PROCESS_ROUTE);
+        }
+        return false;
+    }
+
+    public static boolean isJob(JobInfo job) {
+        if (job != null && job.getProcessItem() != null) {
+            Property p = job.getProcessItem().getProperty();
+            if (p != null) {
+            	return isJob(p);
+            }
+        }
+        return false;
+    }
+
+    public static boolean isJob(Property p) {
+        if (p != null) {
+            return ERepositoryObjectType.getType(p).equals(ERepositoryObjectType.PROCESS);
+        }
+        return false;
+    }
+
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/PomIdsHelper.java

@@ -260,6 +260,12 @@ public class PomIdsHelper {
         return manager.getValue(MavenConstants.POM_FILTER);
     }
 
+    public static boolean useProfileModule() {
+        String projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
+        ProjectPreferenceManager manager = getPreferenceManager(projectTechName);
+        return manager.getBoolean(MavenConstants.USE_PROFILE_MODULE);
+    }
+
     private static String getGroupId(String projectTechName, String baseName, Property property) {
         if (projectTechName == null) {
             projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
@@ -298,6 +304,12 @@ public class PomIdsHelper {
         return PREFIX_DEFAULT_GROUPID + projectName.toLowerCase();
     }
 
+    public static boolean skipFolders() {
+        String projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
+        ProjectPreferenceManager manager = getPreferenceManager(projectTechName);
+        return manager.getBoolean(MavenConstants.SKIP_FOLDERS);
+    }
+
     public static boolean isValidGroupId(String text) {
         if (text != null && text.matches("[\\w\\.]+")) { //$NON-NLS-1$
             return true;

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/PomUtil.java

@@ -27,10 +27,12 @@ import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 
 import javax.xml.parsers.DocumentBuilder;
@@ -78,6 +80,7 @@ import org.talend.core.model.process.JobInfo;
 import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
+import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.nexus.TalendMavenResolver;
 import org.talend.core.runtime.maven.MavenArtifact;
@@ -90,6 +93,7 @@ import org.talend.designer.maven.model.TalendMavenConstants;
 import org.talend.designer.maven.template.MavenTemplateManager;
 import org.talend.designer.maven.tools.ProcessorDependenciesManager;
 import org.talend.designer.runprocess.IProcessor;
+import org.talend.designer.runprocess.IRunProcessService;
 import org.talend.repository.ProjectManager;
 import org.w3c.dom.Attr;
 import org.w3c.dom.DOMImplementation;
@@ -116,7 +120,9 @@ public class PomUtil {
         if (pomFile == null) {
             throw new NullPointerException("the output file is null.");
         }
-
+        Properties properties = new Properties();
+        properties.putAll(model.getProperties());
+        model.setProperties(properties);
         /*
          * copied the codes from createMavenModel of MavenModelManager
          */
@@ -138,7 +144,9 @@ public class PomUtil {
         if (pomFile == null) {
             throw new NullPointerException("the output file is null.");
         }
-
+        Properties properties = new Properties();
+        properties.putAll(model.getProperties());
+        model.setProperties(properties);
         ByteArrayOutputStream buf = new ByteArrayOutputStream();
         MavenPlugin.getMaven().writeModel(model, buf);
 
@@ -1017,4 +1025,34 @@ public class PomUtil {
                     || pathname.getName().equals("m2e-lastUpdated.properties"); //$NON-NLS-1$
         }
     };
+
+    public static boolean checkIfJobDependencyExist(IFile parentJobPom, Property property) throws Exception {
+        boolean found = false;
+        Model model = MavenPlugin.getMavenModelManager().readMavenModel(parentJobPom);
+        String groupId = PomIdsHelper.getJobGroupId(property);
+        String artifactId = PomIdsHelper.getJobArtifactId(property);
+        String version = PomIdsHelper.getJobVersion(property);
+        for (Dependency dependency : model.getDependencies()) {
+            if (dependency.getGroupId().equals(groupId) && dependency.getArtifactId().equals(artifactId)
+                    && dependency.getVersion().equals(version)) {
+                found = true;
+                break;
+            }
+        }
+        return found;
+    }
+
+    public static Set<Dependency> getCodesDependencies(ERepositoryObjectType codeType) throws CoreException {
+        Set<Dependency> dependencies = new HashSet<Dependency>();
+
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
+            IRunProcessService runProcessService = (IRunProcessService) GlobalServiceRegister.getDefault()
+                    .getService(IRunProcessService.class);
+            ITalendProcessJavaProject talendCodeJavaProject = runProcessService.getTalendCodeJavaProject(codeType);
+            IFile projectPom = talendCodeJavaProject.getProjectPom();
+            Model model = MODEL_MANAGER.readMavenModel(projectPom);
+            dependencies.addAll(model.getDependencies());
+        }
+        return dependencies;
+    }
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/SortableDependency.java

@@ -10,4 +10,37 @@ public class SortableDependency extends Dependency implements Comparable<Sortabl
         return getArtifactId().compareTo(o.getArtifactId());
     }
 
+    @Override
+    public int hashCode() {
+        int result = 17;
+        result = 31 * result + (getGroupId() == null ? 0 : getGroupId().hashCode());
+        result = 31 * result + (getArtifactId() == null ? 0 : getArtifactId().hashCode());
+        result = 31 * result + (getVersion() == null ? 0 : getVersion().hashCode());
+        result = 31 * result + (getType() == null ? 0 : getType().hashCode());
+        result = 31 * result + (getClassifier() == null ? 0 : getClassifier().hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (!(obj instanceof SortableDependency)) {
+            return false;
+        }
+        SortableDependency caseobj = (SortableDependency) obj;
+
+        return (getGroupId() == caseobj.getGroupId() || (getGroupId() != null && getGroupId().equals(caseobj.getGroupId())))
+                && (getArtifactId() == caseobj.getArtifactId()
+                        || (getArtifactId() != null && getArtifactId().equals(caseobj.getArtifactId())))
+                && (getVersion() == caseobj.getVersion() || (getVersion() != null && getVersion().equals(caseobj.getVersion())))
+                && (getType() == caseobj.getType() || (getType() != null && getType().equals(caseobj.getType())))
+                && (getClassifier() == caseobj.getClassifier()
+                        || (getClassifier() != null && getClassifier().equals(caseobj.getClassifier())));
+    }
+
 }

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/ui/startup/ShareLibsSynchronizer.java

@@ -14,12 +14,17 @@ package org.talend.librariesmanager.ui.startup;
 
 import org.eclipse.ui.IStartup;
 
+import java.util.logging.Logger;
+import org.talend.librariesmanager.prefs.LibrariesManagerUtils;
+
 /**
  * created by wchen on 2015-6-15 Detailled comment
  *
  */
 public class ShareLibsSynchronizer implements IStartup {
 
+    private static final Logger LOGGER = Logger.getLogger(ShareLibsSynchronizer.class.getCanonicalName());
+
     /*
      * (non-Javadoc)
      * 
@@ -27,8 +32,11 @@ public class ShareLibsSynchronizer implements IStartup {
      */
     @Override
     public void earlyStartup() {
-        ShareLibsJob job = new ShareLibsJob();
-        job.schedule();
+        if (LibrariesManagerUtils.shareLibsAtStartup()) {
+            ShareLibsJob job = new ShareLibsJob();
+            job.schedule();
+        } else {
+            LOGGER.info("Skip sharing libraries");
+        }
     }
-
 }

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/ui/startup/ShareMavenArtifactsOnStartup.java

@@ -20,6 +20,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.log4j.Logger;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.SubMonitor;
 import org.talend.core.GlobalServiceRegister;
@@ -40,6 +41,7 @@ import org.talend.utils.io.FilesUtils;
  */
 public class ShareMavenArtifactsOnStartup extends ShareLibrareisHelper {
 
+    private static final Logger LOGGER = Logger.getLogger(ShareMavenArtifactsOnStartup.class);
     @Override
     public Map<ModuleNeeded, File> getFilesToShare(IProgressMonitor monitor) {
         Map<ModuleNeeded, File> files = new HashMap<ModuleNeeded, File>();
@@ -92,6 +94,7 @@ public class ShareMavenArtifactsOnStartup extends ShareLibrareisHelper {
             FilesUtils.copyFile(generatedPom, pomFile);
             FilesUtils.deleteFolder(generatedPom.getParentFile(), true);
         }
+        LOGGER.info("ShareMavenArtifactsOnStartup, shareToRepository, file: " + file);
         deployer.deploy(file, artifact);
         // artifact.setType(MavenConstants.PACKAGING_POM);
         // deployer.deploy(pomFile, artifact);

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/utils/DownloadModuleRunnable.java

@@ -13,6 +13,7 @@
 package org.talend.librariesmanager.utils;
 
 import java.net.URL;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -47,6 +48,8 @@ abstract public class DownloadModuleRunnable implements IRunnableWithProgress {
 
     private boolean checkLibraries;
 
+    private boolean showErrorInDialog = true;
+
     /**
      * DOC sgandon DownloadModuleRunnable constructor comment.
      * 
@@ -130,6 +133,13 @@ abstract public class DownloadModuleRunnable implements IRunnableWithProgress {
                 downloadFailed.add(module.getName());
             }
         }
+
+        if (showErrorInDialog && !downloadFailed.isEmpty()) {
+            Exception ex = new Exception(Messages.getString("DownloadModuleRunnable.jar.download.failed",
+                    Arrays.toString(downloadFailed.toArray(new String[downloadFailed.size()]))));
+            MessageBoxExceptionHandler.process(ex);
+        }
+
         if (checkLibraries) {
             ILibrariesService librariesService = (ILibrariesService) GlobalServiceRegister.getDefault()
                     .getService(ILibrariesService.class);

main/plugins/org.talend.librariesmanager/resources/java/routines/system/JDBCUtil.java

@@ -58,11 +58,22 @@ public class JDBCUtil {
     }
     
     public static Date getDate(ResultSet rs, int index) throws java.sql.SQLException {
-        if(rs.getTimestamp(index) != null) {
-            return new Date(rs.getTimestamp(index).getTime());
+        Date result = null;
+        try {
+            if(rs.getTimestamp(index) != null) {
+                result = new Date(rs.getTimestamp(index).getTime());
+                return result;
+            }
+        } catch (java.sql.SQLException e) {
         }
-        
-        return null;
+        try {
+            if(rs.getDate(index) != null) {
+                result = new Date(rs.getDate(index).getTime());
+                return result;
+            }
+        } catch (java.sql.SQLException e) {
+        }
+        return result;
     }
     
     //decrease the get method call number

main/plugins/org.talend.librariesmanager/resources/java/routines/system/PasswordEncryptUtil.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2018 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
@@ -75,4 +75,6 @@ public class PasswordEncryptUtil {
         return input;
     }
 
+    public static final String PASSWORD_FOR_LOGS_VALUE = "...";
+
 }

main/plugins/org.talend.librariesmanager/src/main/java/messages.properties

@@ -26,5 +26,6 @@ ModulesNeededProvider.Module.Exception=Needed module list do not contains this m
 
 ShareLibsJob.message=Share libraries from maven to {0} server
 ShareLibsJob.sharingLibraries=Sharing libraries : {0}
+ShareLibsJob.monitor.cancelled=User cancelled.
 
 ExtensionModuleManager.moduleGroup.cycleReference=Module group {0} was cycle referenced!

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/maven/ShareLibrareisHelper.java

@@ -14,25 +14,30 @@ package org.talend.librariesmanager.maven;
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.log4j.Logger;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Status;
 import org.eclipse.core.runtime.SubMonitor;
 import org.eclipse.core.runtime.jobs.Job;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.model.general.ModuleNeeded;
-import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.RepositoryArtifactHandlerManager;
 import org.talend.core.nexus.TalendLibsServerManager;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.librariesmanager.i18n.Messages;
+import org.talend.librariesmanager.model.service.LocalLibraryManager;
+import org.talend.librariesmanager.nexus.utils.ShareLibrariesUtil;
 
 /**
  * created by Talend on 2015年7月31日 Detailled comment
@@ -42,6 +47,7 @@ public abstract class ShareLibrareisHelper {
 
     private final String TYPE_NEXUS = "nexus";
 
+    private final Logger LOGGER = Logger.getLogger(ShareLibrareisHelper.class);
     protected MavenArtifactsHandler deployer = new MavenArtifactsHandler();
 
     public IStatus shareLibs(Job job, IProgressMonitor monitor) {
@@ -49,80 +55,126 @@ public abstract class ShareLibrareisHelper {
         IStatus status = Status.OK_STATUS;
         // deploy to maven if needed and share to custom nexus
         try {
-            int searchLimit = 50;
             setJobName(job, Messages.getString("ShareLibsJob.message", TYPE_NEXUS));
-            final List<MavenArtifact> searchResults = new ArrayList<MavenArtifact>();
             ArtifactRepositoryBean customNexusServer = TalendLibsServerManager.getInstance().getCustomNexusServer();
             IRepositoryArtifactHandler customerRepHandler = RepositoryArtifactHandlerManager
                     .getRepositoryHandler(customNexusServer);
             if (customerRepHandler != null) {
                 filesToShare = getFilesToShare(monitor);
+                LOGGER.info("filesToShare size: " + (filesToShare == null ? 0 : filesToShare.size()));
                 if (filesToShare == null) {
                     return Status.CANCEL_STATUS;
                 }
-                SubMonitor mainSubMonitor = SubMonitor.convert(monitor, filesToShare.size());
 
                 // collect groupId to search
                 Set<String> groupIds = new HashSet<String>();
+                Map<String, List<MavenArtifact>> snapshotArtifactMap = new HashMap<String, List<MavenArtifact>>();
+                Map<String, List<MavenArtifact>> releaseArtifactMap = new HashMap<String, List<MavenArtifact>>();
+                Set<String> snapshotGroupIdSet = new HashSet<String>();
+                Set<String> releaseGroupIdSet = new HashSet<String>();
                 for (ModuleNeeded module : filesToShare.keySet()) {
+                    checkCancel(monitor);
                     if (module.getMavenUri() != null) {
                         MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(module.getMavenUri());
                         if (parseMvnUrl != null) {
                             groupIds.add(parseMvnUrl.getGroupId());
+                            if (isSnapshotVersion(parseMvnUrl.getVersion())) {
+                                snapshotGroupIdSet.add(parseMvnUrl.getGroupId());
+                            } else {
+                                releaseGroupIdSet.add(parseMvnUrl.getGroupId());
+                            }
                         }
                     }
                 }
+                LOGGER.info("snapshotGroupIdSet size: " + snapshotGroupIdSet.size() + ", releaseGroupIdSet size: "
+                        + releaseGroupIdSet.size());
+                List<MavenArtifact> searchResults = new ArrayList<MavenArtifact>();
                 for (String groupId : groupIds) {
-                    searchResults.addAll(customerRepHandler.search(groupId, null, null, true, true));
+                    if (releaseGroupIdSet.contains(groupId)) {
+                        searchResults = customerRepHandler.search(groupId, null, null, true, false);
+                        LOGGER.info("release searchResults size: " + (searchResults == null ? 0 : searchResults.size())
+                                + ", groupId: "
+                                + groupId);
+                        if (searchResults != null) {
+                            for (MavenArtifact result : searchResults) {
+                                checkCancel(monitor);
+                                ShareLibrariesUtil.putArtifactToMap(result, releaseArtifactMap, false);
+                            }
+                        }
+                    }
+                    if (snapshotGroupIdSet.contains(groupId)) {
+                        searchResults = customerRepHandler.search(groupId, null, null, false, true);
+                        LOGGER.info("snapshot searchResults size: " + (searchResults == null ? 0 : searchResults.size())
+                                + ", groupId: " + groupId);
+                        if (searchResults != null) {
+                            for (MavenArtifact result : searchResults) {
+                                checkCancel(monitor);
+                                ShareLibrariesUtil.putArtifactToMap(result, snapshotArtifactMap, true);
+                            }
+                        }
+                    }
                 }
-
-                int limit = searchLimit;
-                int shareIndex = 0;
                 Iterator<ModuleNeeded> iterator = filesToShare.keySet().iterator();
+                Map<File, MavenArtifact> shareFiles = new HashMap<>();
                 while (iterator.hasNext()) {
                     if (monitor.isCanceled()) {
                         return Status.CANCEL_STATUS;
                     }
-                    shareIndex++;
-                    if (shareIndex == limit) {
-                        limit += searchLimit;
-                    }
-
                     ModuleNeeded next = iterator.next();
                     File file = filesToShare.get(next);
-                    String name = file.getName();
                     MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(next.getMavenUri());
                     if (artifact == null) {
                         continue;
                     }
-
-                    boolean eixst = false;
-                    String groupId = artifact.getGroupId();
-                    String artifactId = artifact.getArtifactId();
-                    String version = artifact.getVersion();
-                    for (MavenArtifact remoteAtifact : searchResults) {
-                        String rGroup = remoteAtifact.getGroupId();
-                        String rArtifact = remoteAtifact.getArtifactId();
-                        String rVersion = remoteAtifact.getVersion();
-                        if (groupId != null && artifactId != null && version != null && groupId.equals(rGroup)
-                                && artifactId.equals(rArtifact) && version.equals(rVersion)) {
-                            eixst = true;
-                            break;
+                    // If from custom component definition file
+                    if (LocalLibraryManager.isSystemCacheFile(file.getName())
+                            || (LocalLibraryManager.isComponentDefinitionFileType(file.getName())
+                                    && isTalendLibraryGroupId(artifact))) {
+                        continue;
+                    }
+                    try {
+                        Integer.parseInt(artifact.getType());
+                        // FIXME unexpected type if it's an integer, should fix it in component module definition.
+                        continue;
+                    } catch (NumberFormatException e) {
+                        //
+                    }
+                    boolean isSnapshotVersion = isSnapshotVersion(artifact.getVersion());
+                    String key = ShareLibrariesUtil.getArtifactKey(artifact, isSnapshotVersion);
+                    List<MavenArtifact> artifactList = null;
+                    if (isSnapshotVersion) {
+                        artifactList = snapshotArtifactMap.get(key);
+                    } else {
+                        artifactList = releaseArtifactMap.get(key);
+                        // skip checksum for release artifact.
+                        if (artifactList != null && artifactList.contains(artifact)
+                                && !Boolean.getBoolean("force_libs_release_update")) {
+                            continue;
                         }
                     }
-                    if (eixst) {
-                        continue;
+                    if (artifactList != null && artifactList.size() > 0) {
+                        if (ShareLibrariesUtil.isSameFileWithRemote(file, artifactList, customNexusServer, customerRepHandler,
+                                isSnapshotVersion)) {
+                            continue;
+                        }
                     }
-                    mainSubMonitor.setTaskName(Messages.getString("ShareLibsJob.sharingLibraries", name));
-
+                    shareFiles.put(file, artifact);
+                }
+                LOGGER.info("shareFiles size: " + shareFiles.size());
+                SubMonitor mainSubMonitor = SubMonitor.convert(monitor, shareFiles.size());
+                shareFiles.forEach((k, v) -> {
                     try {
-                        shareToRepository(file, artifact);
+                        mainSubMonitor.setTaskName(Messages.getString("ShareLibsJob.sharingLibraries", k.getName()));
+                        shareToRepository(k, v);
                         mainSubMonitor.worked(1);
                     } catch (Exception e) {
-                        continue;
+                        ExceptionHandler.process(e);
                     }
-                }
+                });
             }
+        } catch (InterruptedException e) {
+            ExceptionHandler.process(e);
+            status = Status.CANCEL_STATUS;
         } catch (Exception e) {
             status = new Status(IStatus.ERROR, "unknown", IStatus.ERROR, "Share libraries failed !", e);
         }
@@ -131,12 +183,31 @@ public abstract class ShareLibrareisHelper {
 
     }
 
+    private boolean isTalendLibraryGroupId(MavenArtifact artifact) {
+        if ("org.talend.libraries".equalsIgnoreCase(artifact.getGroupId())) {
+            return true;
+        }
+        return false;
+    }
+
+    private boolean isSnapshotVersion(String version) {
+        if (version != null && version.toUpperCase().endsWith(MavenUrlHelper.VERSION_SNAPSHOT)) {
+            return true;
+        }
+        return false;
+    }
     private void setJobName(Job job, String jobName) {
         if (job != null) {
             job.setName(jobName);
         }
     }
 
+    protected void checkCancel(IProgressMonitor monitor) throws InterruptedException {
+        if (monitor.isCanceled()) {
+            throw new InterruptedException(Messages.getString("ShareLibsJob.monitor.cancelled"));
+        }
+    }
+
     public abstract Map<ModuleNeeded, File> getFilesToShare(IProgressMonitor monitor);
 
     public abstract void shareToRepository(File jarFile, MavenArtifact module) throws Exception;

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/ModulesNeededProvider.java

@@ -397,6 +397,7 @@ public class ModulesNeededProvider {
         List<String> list = getInstallURL(importType);
         ModuleNeeded moduleNeeded = new ModuleNeeded(context, importType.getMODULE(), msg, importType.isREQUIRED(), list,
                 importType.getREQUIREDIF(), importType.getMVN());
+        moduleNeeded.setUseReleaseVersion(true);
         initBundleID(importType, moduleNeeded);
         moduleNeeded.setMrRequired(importType.isMRREQUIRED());
         moduleNeeded.setShow(importType.isSHOW());
@@ -543,9 +544,9 @@ public class ModulesNeededProvider {
                             Property property = findRoutinesPropery(infor.getId(), infor.getName(), routines, type);
                             if (property != null) {
                                 if (((RoutineItem) property.getItem()).isBuiltIn()) {
-                                    systemRoutines.add(infor.getId());
+                                    systemRoutines.add(property.getId());
                                 } else {
-                                    userRoutines.add(infor.getId());
+                                    userRoutines.add(property.getId());
                                 }
                             }
 
@@ -820,6 +821,7 @@ public class ModulesNeededProvider {
         uripath = ExtensionModuleManager.getInstance().getFormalModulePath(uripath, current);
         String mvn_rui = current.getAttribute(ExtensionModuleManager.MVN_URI_ATTR);
         ModuleNeeded module = new ModuleNeeded(context, name, message, required);
+        module.setUseReleaseVersion(true);
         if (uripath != null && libManagerService.checkJarInstalledFromPlatform(uripath)) {
             module.setModuleLocaion(uripath);
         }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/LocalLibraryManager.java

@@ -29,9 +29,12 @@ import java.util.Collection;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Stream;
 
 import javax.xml.bind.DatatypeConverter;
 
@@ -61,10 +64,10 @@ import org.talend.core.model.components.ComponentProviderInfo;
 import org.talend.core.model.components.IComponent;
 import org.talend.core.model.components.IComponentsService;
 import org.talend.core.model.general.ILibrariesService;
-import org.talend.core.model.general.ILibrariesService.IChangedLibrariesListener;
 import org.talend.core.model.general.ModuleNeeded;
-import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
 import org.talend.core.model.general.ModuleStatusProvider;
+import org.talend.core.model.general.ILibrariesService.IChangedLibrariesListener;
+import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
 import org.talend.core.nexus.ArtifactRepositoryBean;
 import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.NexusServerUtils;
@@ -80,6 +83,7 @@ import org.talend.designer.maven.utils.PomUtil;
 import org.talend.librariesmanager.maven.MavenArtifactsHandler;
 import org.talend.librariesmanager.model.ExtensionModuleManager;
 import org.talend.librariesmanager.model.ModulesNeededProvider;
+import org.talend.librariesmanager.nexus.utils.ShareLibrariesUtil;
 import org.talend.librariesmanager.prefs.LibrariesManagerUtils;
 import org.talend.osgi.hook.notification.JarMissingObservable;
 
@@ -108,7 +112,17 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
     private JarMissingObservable missingJarObservable;
 
     private MavenArtifactsHandler deployer;
-
+    
+    private static final List<String> COMPONENT_DEFINITION_FILE_TYPE_LIST = new ArrayList<String>() {
+        {
+            add(".javajet");
+            add(".png");
+            add(".jpg");
+            add("_java.xml");
+            add(".properties");
+            add(".txt");
+        }
+    };
     /**
      * DOC nrousseau LocalLibraryManager constructor comment.
      */
@@ -155,7 +169,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         if (file == null || !file.exists()) {
             return;
         }
-        install(file, mavenUri, updateNexusJar, monitorWrap);
+        install(file, mavenUri, updateNexusJar, false, monitorWrap);
     }
 
     /**
@@ -166,7 +180,8 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
      * @param mavenUri snaopshot mvn uri
      * @param monitorWrap
      */
-    private void install(File file, String mavenRUI, boolean updateRemoteJar, IProgressMonitor... monitorWrap) {
+    private void install(File file, String mavenRUI, boolean updateRemoteJar, boolean useReleaseVersion,
+            IProgressMonitor... monitorWrap) {
         try {
             if (file.isDirectory()) {
                 List<File> jarFiles = FilesUtils.getJarFilesFromFolder(file, null);
@@ -174,7 +189,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
                 if (!jarFiles.isEmpty()) {
                     for (File jarFile : jarFiles) {
                         if (mavenRUI == null) {
-                            guessMavenRUIFromIndex(jarFile, sourceAndMavenUri);
+                            guessMavenRUIFromIndex(jarFile, useReleaseVersion, sourceAndMavenUri);
                         } else {
                             sourceAndMavenUri.put(mavenRUI, jarFile.getAbsolutePath());
                         }
@@ -186,7 +201,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
             } else {
                 Map<String, String> sourceAndMavenUri = new HashMap<String, String>();
                 if (mavenRUI == null) {
-                    guessMavenRUIFromIndex(file, sourceAndMavenUri);
+                    guessMavenRUIFromIndex(file, useReleaseVersion, sourceAndMavenUri);
                 } else {
                     sourceAndMavenUri.put(mavenRUI, file.getAbsolutePath());
                 }
@@ -201,7 +216,19 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         }
     }
 
-    private void guessMavenRUIFromIndex(File jarFile, Map<String, String> sourceAndMavenUri) {
+    public void guessMavenRUIFromIndex(File jarFile, Map<String, String> sourceAndMavenUri) {
+        guessMavenRUIFromIndex(jarFile, false, sourceAndMavenUri);
+
+    }
+
+    /**
+     * 
+     * DOC wchen Comment method "guessMavenRUIFromIndex".
+     * 
+     * @param jarFile jar file to guess maven url
+     * @param useReleaseVersion generate release version if not find from index
+     */
+    private void guessMavenRUIFromIndex(File jarFile, boolean useReleaseVersion, Map<String, String> sourceAndMavenUri) {
         // TODO????? should deploy with all versions
         String urisFromIndex = LibrariesIndexManager.getInstance().getMavenLibIndex().getJarsToRelativePath()
                 .get(jarFile.getName());
@@ -226,7 +253,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         // deploy as defaultMavenUri in case jar name is diffrent from artifactId in mvnuri from
         // index
         if (deployAsDefault) {
-            String defaultMavenUri = MavenUrlHelper.generateMvnUrlForJarName(jarFile.getName());
+            String defaultMavenUri = MavenUrlHelper.generateMvnUrlForJarName(jarFile.getName(), true, !useReleaseVersion);
             String customMavenURI = getCustomMavenURI(defaultMavenUri);
             if (customMavenURI != null) {
                 defaultMavenUri = customMavenURI;
@@ -356,6 +383,24 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         return false;
     }
 
+    public static boolean isComponentDefinitionFileType(String fileName) {
+        if (fileName != null) {
+            for (String type : COMPONENT_DEFINITION_FILE_TYPE_LIST) {
+                if (fileName.toLowerCase().endsWith(type)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    public static boolean isSystemCacheFile(String fileName) {
+        if ("Thumbs.db".equals(fileName)) {
+            return true;
+        }
+        return false;
+    }
+
     private File retrieveJarFromLocal(ModuleNeeded module) {
         File jarFile = null;
         Set<String> toResolve = guessMavenURI(module);
@@ -825,7 +870,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
                     ExceptionHandler.log("missing jar:" + module.getModuleName());
                 }
                 if (fileToDeploy != null) {
-                    install(fileToDeploy, mavenUri, false, monitorWrap);
+                    install(fileToDeploy, mavenUri, false, false, monitorWrap);
                 }
             }
         }
@@ -919,6 +964,9 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         if (ModuleStatusProvider.getDeployStatus(mvnUriStatusKey) == ELibraryInstallStatus.NOT_DEPLOYED) {
             return null;
         }
+        if(mvnUriStatusKey == null){
+            return null;
+        }
         // then try to resolve locally
         String localMavenUri = mvnUriStatusKey.replace("mvn:", "mvn:" + MavenConstants.LOCAL_RESOLUTION_URL + "!"); //$NON-NLS-1$ //$NON-NLS-2$
         try {
@@ -1117,51 +1165,35 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         }
 
         if (service != null) {
-            deployLibsFromComponentFolder(service, platformURLMap);
+            calculateModulesIndexFromComponentFolder(service, platformURLMap);
         }
 
         saveMavenIndex(mavenURIMap, monitorWrap);
         savePlatfromURLIndex(platformURLMap, monitorWrap);
+        
+        if (service != null) {
+            deployLibsFromCustomComponents(service, platformURLMap);
+        }
 
     }
 
     /**
-     * 
+     *
      * The old components might use some jars in component folder and theres jars are not configured with platfrom URL
-     * 
+     *
      * @param service
      * @param libsWithoutUri
      * @param platformURLMap
      */
-    private void deployLibsFromComponentFolder(IComponentsService service, Map<String, String> platformURLMap) {
-        Set<File> needToDeploy = new HashSet<File>();
+    private void calculateModulesIndexFromComponentFolder(IComponentsService service, Map<String, String> platformURLMap) {
         List<ComponentProviderInfo> componentsFolders = service.getComponentsFactory().getComponentsProvidersInfo();
         for (ComponentProviderInfo providerInfo : componentsFolders) {
             String contributeID = providerInfo.getContributer();
             String id = providerInfo.getId();
             try {
-                File file = new File(providerInfo.getLocation());
-                if ("org.talend.designer.components.model.UserComponentsProvider".equals(id)
-                        || "org.talend.designer.components.exchange.ExchangeComponentsProvider".equals(id)) {
-                    if (file.isDirectory()) {
-                        List<File> jarFiles = FilesUtils.getJarFilesFromFolder(file, null);
-                        if (jarFiles.size() > 0) {
-                            for (File jarFile : jarFiles) {
-                                String name = jarFile.getName();
-                                if (platformURLMap.get(name) != null) {
-                                    continue;
-                                }
-                                needToDeploy.add(jarFile);
-                            }
-                        }
-                    } else {
-                        if (platformURLMap.get(file.getName()) != null) {
-                            continue;
-                        }
-                        needToDeploy.add(file);
-                    }
-                } else {
-                    // for other component provider ,add jars to the platform url index
+                if (!"org.talend.designer.components.model.UserComponentsProvider".equals(id)
+                        && !"org.talend.designer.components.exchange.ExchangeComponentsProvider".equals(id)) {
+                    File file = new File(providerInfo.getLocation());
                     List<File> jarFiles = FilesUtils.getJarFilesFromFolder(file, null, "ext");
                     if (jarFiles.size() > 0) {
                         for (File jarFile : jarFiles) {
@@ -1191,41 +1223,156 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
                 continue;
             }
         }
+    }
 
-        // deploy needed jars for User and Exchange component providers
-        if (!needToDeploy.isEmpty()) {
-            // search on nexus to avoid deploy the jar many times
-            Set<File> existFiles = new HashSet<File>();
-            ArtifactRepositoryBean customNexusServer = TalendLibsServerManager.getInstance().getCustomNexusServer();
-            IRepositoryArtifactHandler customerRepHandler = RepositoryArtifactHandlerManager
-                    .getRepositoryHandler(customNexusServer);
-            if (customerRepHandler != null) {
-                List<MavenArtifact> searchResult = new ArrayList<>();
+    private void deployLibsFromCustomComponents(IComponentsService service, Map<String, String> platformURLMap) {
+        boolean deployToRemote = true;
+        if (!LibrariesManagerUtils.shareLibsAtStartup()) {
+            log.info("Skip deploying libs from custom components");
+            deployToRemote = false;
+        }
+        Set<File> needToDeploy = new HashSet<>();
+        List<ComponentProviderInfo> componentsFolders = service.getComponentsFactory().getComponentsProvidersInfo();
+        for (ComponentProviderInfo providerInfo : componentsFolders) {
+            String id = providerInfo.getId();
+            try {
+                File file = new File(providerInfo.getLocation());
+                if ("org.talend.designer.components.model.UserComponentsProvider".equals(id)
+                        || "org.talend.designer.components.exchange.ExchangeComponentsProvider".equals(id)) {
+                    if (file.isDirectory()) {
+                        List<File> jarFiles = FilesUtils.getJarFilesFromFolder(file, null);
+                        if (jarFiles.size() > 0) {
+                            for (File jarFile : jarFiles) {
+                                String name = jarFile.getName();
+                                if (!canDeployFromCustomComponentFolder(name)
+                                        || platformURLMap.get(name) != null) {
+                                    continue;
+                                }
+                                needToDeploy.add(jarFile);
+                            }
+                        }
+                    } else {
+                        if (platformURLMap.get(file.getName()) != null) {
+                            continue;
+                        }
+                        needToDeploy.add(file);
+                    }
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+                continue;
+            }
+
+            if (!deployToRemote) {
+                needToDeploy.forEach(libFile -> {
+                    try {
+                        // install as release version if can't find mvn url from index
+                        install(libFile, null, false, true);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
+                });
+
+                return;
+            }
+
+            // deploy needed jars for User and Exchange component providers
+            Map<String, List<MavenArtifact>> snapshotArtifactMap = new HashMap<String, List<MavenArtifact>>();
+            Map<String, List<MavenArtifact>> releaseArtifactMap = new HashMap<String, List<MavenArtifact>>();
+            if (!needToDeploy.isEmpty()) {
+                // search on nexus to avoid deploy the jar many times
+                Set<File> existFiles = new HashSet<>();
+                ArtifactRepositoryBean customNexusServer = TalendLibsServerManager.getInstance().getCustomNexusServer();
+                IRepositoryArtifactHandler customerRepHandler = RepositoryArtifactHandlerManager
+                        .getRepositoryHandler(customNexusServer);
+                List<MavenArtifact> snapshotResult = new ArrayList<>();
+                List<MavenArtifact> releaseResult = new ArrayList<>();
                 try {
-                    searchResult = customerRepHandler.search(MavenConstants.DEFAULT_LIB_GROUP_ID, null, null, true, true);
+                    snapshotResult = customerRepHandler.search(MavenConstants.DEFAULT_LIB_GROUP_ID, null, null, false, true);
+                    if (snapshotResult != null) {
+                        for (MavenArtifact result : snapshotResult) {
+                            ShareLibrariesUtil.putArtifactToMap(result, snapshotArtifactMap, true);
+                        }
+                    }
+                    releaseResult = customerRepHandler.search(MavenConstants.DEFAULT_LIB_GROUP_ID, null, null, true, false);
+                    if (releaseResult != null) {
+                        for (MavenArtifact result : releaseResult) {
+                            ShareLibrariesUtil.putArtifactToMap(result, releaseArtifactMap, false);
+                        }
+                    }
                 } catch (Exception e) {
                     ExceptionHandler.process(e);
                 }
-                for (MavenArtifact artifact : searchResult) {
-                    for (File file : needToDeploy) {
-                        if (artifact.getFileName().equals(file.getName())) {
-                            existFiles.add(file);
+                for (File exsitFile : needToDeploy) {
+                    if (customerRepHandler != null) {
+                        try {
+                            String name = exsitFile.getName();
+                            String mvnUrlSnapshot = MavenUrlHelper.generateMvnUrlForJarName(name, true, true);
+                            MavenArtifact artifactSnapshot = MavenUrlHelper.parseMvnUrl(mvnUrlSnapshot);
+                            String keySnapshot = ShareLibrariesUtil.getArtifactKey(artifactSnapshot, true);
+                            List<MavenArtifact> artifactListSnapshot = null;
+                            artifactListSnapshot = snapshotArtifactMap.get(keySnapshot);
+                            // snapshot
+                            if (artifactListSnapshot != null && artifactListSnapshot.size() > 0) {
+                                if (ShareLibrariesUtil.isSameFileWithRemote(exsitFile, artifactListSnapshot, customNexusServer,
+                                        customerRepHandler, true)) {
+                                    existFiles.add(exsitFile);
+                                }
+                            }
+                            // release
+                            String mvnUrlRelease = MavenUrlHelper.generateMvnUrlForJarName(name, true, false);
+                            MavenArtifact artifactRelease = MavenUrlHelper.parseMvnUrl(mvnUrlRelease);
+                            String keyRelease = ShareLibrariesUtil.getArtifactKey(artifactRelease, false);
+                            List<MavenArtifact> artifactListRelease = null;
+                            artifactListRelease = releaseArtifactMap.get(keyRelease);
+                            if (artifactListRelease != null && artifactListRelease.size() > 0) {
+                                if (ShareLibrariesUtil.isSameFileWithRemote(exsitFile, artifactListRelease, customNexusServer,
+                                        customerRepHandler, false)) {
+                                    existFiles.add(exsitFile);
+                                }
+                            }
+                        } catch (Exception e) {
+                            ExceptionHandler.process(e);
                         }
                     }
                 }
-
-            }
-            needToDeploy.removeAll(existFiles);
-            for (File file : needToDeploy) {
-                try {
-                    deploy(file.toURI());
-                } catch (Exception e) {
-                    ExceptionHandler.process(e);
+                needToDeploy.removeAll(existFiles);
+                // check sha code to avoid same jar in diff component depoly multi times
+                Map<String, File> shaMap = new HashMap<>();
+                Iterator<File> it = needToDeploy.iterator();
+                while (it.hasNext()) {
+                    try {
+                        File dupFile = it.next();
+                        String localFileShaCode = DigestUtils.shaHex(new FileInputStream(dupFile));
+                        if (shaMap.get(localFileShaCode) == null) {
+                            shaMap.put(localFileShaCode, dupFile);
+                        } else {
+                            it.remove();
+                        }
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
+                }
+                for (File file : needToDeploy) {
+                    try {
+                        // deploy as release version if can't find mvn url from index
+                        install(file, null, true, true);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                        continue;
+                    }
                 }
             }
         }
     }
 
+    private boolean canDeployFromCustomComponentFolder(String fileName) {
+        if (isSystemCacheFile(fileName) || isComponentDefinitionFileType(fileName)) {
+            return false;
+        }
+        return true;
+    }
+
     private void warnDuplicated(List<ModuleNeeded> modules, Set<String> duplicates, String type) {
         for (String lib : duplicates) {
             Set<String> components = new HashSet<String>();
@@ -1494,7 +1641,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         return isSameFile(jarFile, jarInLib);
     }
 
-    private boolean isSameFile(File f1, File f2) {
+    public boolean isSameFile(File f1, File f2) {
         if (f1 == null || f2 == null) {
             return false;
         }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/AbstractArtifactRepositoryHandler.java

@@ -13,13 +13,18 @@
 package org.talend.librariesmanager.nexus;
 
 import java.io.File;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.util.Dictionary;
 import java.util.Hashtable;
 
+import org.apache.commons.lang.StringUtils;
 import org.talend.core.nexus.ArtifactRepositoryBean;
 import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.NexusConstants;
 import org.talend.core.nexus.TalendMavenResolver;
+import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.utils.string.StringUtilities;
 
 /**
@@ -62,10 +67,22 @@ public abstract class AbstractArtifactRepositoryHandler implements IRepositoryAr
         if (props == null) {
             props = new Hashtable<String, String>();
         }
-        String repositories = null;
-        String custom_server = serverBean.getServer();
+
         String custom_user = serverBean.getUserName();
         String custom_pass = serverBean.getPassword();
+        try {
+            if (StringUtils.isNotBlank(custom_user)) {
+                custom_user = URLEncoder.encode(custom_user, StandardCharsets.UTF_8.toString());
+            }
+            if (StringUtils.isNotBlank(custom_pass)) {
+                custom_pass = URLEncoder.encode(custom_pass, StandardCharsets.UTF_8.toString());
+            }
+        } catch (UnsupportedEncodingException e1) {
+            throw new RuntimeException(e1);
+        }
+
+        String repositories = null;
+        String custom_server = serverBean.getServer();
         String release_rep = serverBean.getRepositoryId();
         String snapshot_rep = serverBean.getSnapshotRepId();
         if (custom_server.endsWith(NexusConstants.SLASH)) {
@@ -83,7 +100,8 @@ public abstract class AbstractArtifactRepositoryHandler implements IRepositoryAr
             repositories = releaseUrl;
         }
         if (snapshot_rep != null) {
-            String snapshotUrl = custom_server + snapshot_rep + "@id=" + snapshot_rep + NexusConstants.SNAPSHOTS;//$NON-NLS-1$
+            String snapshotUrl = custom_server + snapshot_rep + "@id=" + snapshot_rep + NexusConstants.SNAPSHOTS //$NON-NLS-1$
+                    + NexusConstants.DISALLOW_RELEASES;
             if (repositories != null) {
                 repositories = repositories + "," + snapshotUrl;
             } else {
@@ -160,4 +178,10 @@ public abstract class AbstractArtifactRepositoryHandler implements IRepositoryAr
 
     @Override
     public abstract IRepositoryArtifactHandler clone();
+    
+    @Override
+    public String resolveRemoteSha1(MavenArtifact artifact, boolean fromRelease) throws Exception {
+        return artifact.getSha1();
+    }
+        
 }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/ArtifacoryRepositoryHandler.java

@@ -14,6 +14,7 @@ package org.talend.librariesmanager.nexus;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -25,11 +26,15 @@ import org.apache.http.client.fluent.Request;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.params.CoreConnectionPNames;
 import org.apache.http.util.EntityUtils;
 import org.eclipse.m2e.core.MavenPlugin;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.TalendProxySelector.IProxySelectorProvider;
+import org.talend.core.nexus.HttpClientTransport;
 import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.NexusConstants;
+import org.talend.core.nexus.NexusServerUtils;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.designer.maven.aether.RepositorySystemFactory;
@@ -43,9 +48,7 @@ import net.sf.json.JSONObject;
  */
 public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandler {
 
-    private String SEARCH_SERVICE = "api/search/gavc?";
-
-    private String SEARCH_RESULT_PREFIX = "api/storage/";
+    private String SEARCH_SERVICE = "api/search/gavc?"; //$NON-NLS-1$
 
     /*
      * (non-Javadoc)
@@ -80,17 +83,30 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
     }
 
     private boolean doConnectionCheck(String repositoryUrl) throws ClientProtocolException, IOException {
-        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword();
-        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes()));
-        Header authority = new BasicHeader("Authorization", basicAuth);
+        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword(); //$NON-NLS-1$
+        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes())); //$NON-NLS-1$
+        Header authority = new BasicHeader("Authorization", basicAuth); //$NON-NLS-1$
         HttpGet get = new HttpGet(repositoryUrl);
         get.addHeader(authority);
         DefaultHttpClient httpclient = new DefaultHttpClient();
-        HttpResponse response = httpclient.execute(get);
-        if (response.getStatusLine().getStatusCode() == 200) {
-            return true;
+        httpclient.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, NexusServerUtils.getTimeout());
+        httpclient.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, NexusServerUtils.getTimeout());
+        IProxySelectorProvider proxySelector = null;
+        try {
+            try {
+                proxySelector = HttpClientTransport.addProxy(httpclient, new URI(repositoryUrl));
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+            HttpResponse response = httpclient.execute(get);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                return true;
+            }
+            return false;
+        } finally {
+            HttpClientTransport.removeProxy(proxySelector);
+            httpclient.getConnectionManager().shutdown();
         }
-        return false;
     }
 
     /*
@@ -103,51 +119,53 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
     public List<MavenArtifact> search(String groupIdToSearch, String artifactId, String versionToSearch, boolean fromRelease,
             boolean fromSnapshot) throws Exception {
         String serverUrl = serverBean.getServer();
-        if (!serverUrl.endsWith("/")) {
-            serverUrl = serverUrl + "/";
+        if (!serverUrl.endsWith("/")) { //$NON-NLS-1$
+            serverUrl = serverUrl + "/"; //$NON-NLS-1$
         }
         String searchUrl = serverUrl + SEARCH_SERVICE;
 
-        String repositoryId = "";
+        String repositoryId = ""; //$NON-NLS-1$
         if (fromRelease) {
             repositoryId = serverBean.getRepositoryId();
         }
         if (fromSnapshot) {
-            if ("".equals(repositoryId)) {
+            if ("".equals(repositoryId)) { //$NON-NLS-1$
                 repositoryId = serverBean.getSnapshotRepId();
             } else {
-                repositoryId = repositoryId + "," + serverBean.getSnapshotRepId();
+                repositoryId = repositoryId + "," + serverBean.getSnapshotRepId(); //$NON-NLS-1$
             }
         }
         String query = "";//$NON-NLS-1$
-        if (!"".equals(repositoryId)) {
+        if (!"".equals(repositoryId)) { //$NON-NLS-1$
             query = "repos=" + repositoryId;//$NON-NLS-1$
         }
         if (groupIdToSearch != null) {
-            if (!"".equals(query)) {
-                query = query + "&";
+            if (!"".equals(query)) { //$NON-NLS-1$
+                query = query + "&"; //$NON-NLS-1$
             }
             query = query + "g=" + groupIdToSearch;//$NON-NLS-1$
         }
         if (artifactId != null) {
-            if (!"".equals(query)) {
-                query = query + "&";
+            if (!"".equals(query)) { //$NON-NLS-1$
+                query = query + "&"; //$NON-NLS-1$
             }
             query = query + "a=" + artifactId;//$NON-NLS-1$
         }
 
         if (versionToSearch != null) {
-            if (!"".equals(query)) {
-                query = query + "&";
+            if (!"".equals(query)) { //$NON-NLS-1$
+                query = query + "&"; //$NON-NLS-1$
             }
             query = query + "v=" + versionToSearch;//$NON-NLS-1$
         }
         searchUrl = searchUrl + query;
         Request request = Request.Get(searchUrl);
-        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword();
-        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes()));
-        Header authority = new BasicHeader("Authorization", basicAuth);
+        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword(); //$NON-NLS-1$
+        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes())); //$NON-NLS-1$
+        Header authority = new BasicHeader("Authorization", basicAuth); //$NON-NLS-1$
         request.addHeader(authority);
+        Header resultDetailHeader = new BasicHeader("X-Result-Detail", "info"); //$NON-NLS-1$ //$NON-NLS-2$
+        request.addHeader(resultDetailHeader);
         List<MavenArtifact> resultList = new ArrayList<MavenArtifact>();
 
         HttpResponse response = request.execute().returnResponse();
@@ -155,58 +173,67 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
         if (content.isEmpty()) {
             return resultList;
         }
-        JSONObject responseObject = new JSONObject().fromObject(content);
-        String resultStr = responseObject.getString("results");
+        JSONObject responseObject = JSONObject.fromObject(content);
+        String resultStr = responseObject.getString("results"); //$NON-NLS-1$
         JSONArray resultArray = null;
         try {
-            resultArray = new JSONArray().fromObject(resultStr);
+            resultArray = JSONArray.fromObject(resultStr);
         } catch (Exception e) {
             throw new Exception(resultStr);
         }
         if (resultArray != null) {
-            String resultUrl = serverUrl + SEARCH_RESULT_PREFIX;
             for (int i = 0; i < resultArray.size(); i++) {
                 JSONObject jsonObject = resultArray.getJSONObject(i);
-                String uri = jsonObject.getString("uri");
-                uri = uri.substring(resultUrl.length(), uri.length());
-                String[] split = uri.split("/");
+                String artifactPath = jsonObject.getString("path"); //$NON-NLS-1$
+                String[] split = artifactPath.split("/"); //$NON-NLS-1$
+                String lastUpdated = jsonObject.getString("lastUpdated"); //$NON-NLS-1$
                 if (split.length > 4) {
                     String fileName = split[split.length - 1];
-                    if (!fileName.endsWith("pom")) {
+                    if (!fileName.endsWith("pom")) { //$NON-NLS-1$
                         String type = null;
                         int dotIndex = fileName.lastIndexOf('.');
                         if (dotIndex > 0) {
                             type = fileName.substring(dotIndex + 1);
-
                         }
                         if (type != null) {
                             MavenArtifact artifact = new MavenArtifact();
                             String v = split[split.length - 2];
                             String a = split[split.length - 3];
-                            String g = "";
+                            String g = ""; //$NON-NLS-1$
                             for (int j = 1; j < split.length - 3; j++) {
-                                if ("".equals(g)) {
+                                if ("".equals(g)) { //$NON-NLS-1$
                                     g = split[j];
                                 } else {
-                                    g = g + "." + split[j];
+                                    g = g + "." + split[j]; //$NON-NLS-1$
                                 }
                             }
                             artifact.setGroupId(g);
                             artifact.setArtifactId(a);
                             artifact.setVersion(v);
                             artifact.setType(type);
+                            artifact.setLastUpdated(lastUpdated);
+                            fillChecksumData(jsonObject, artifact);
                             resultList.add(artifact);
                         }
                     }
-
                 }
-
             }
         }
-
         return resultList;
     }
 
+    private void fillChecksumData(JSONObject responseObject, MavenArtifact artifact) {
+        if (responseObject.containsKey("checksums")) { //$NON-NLS-1$
+            JSONObject checkSum = responseObject.getJSONObject("checksums"); //$NON-NLS-1$
+            if (checkSum != null && checkSum.containsKey("sha1")) { //$NON-NLS-1$
+                artifact.setSha1(checkSum.getString("sha1")); //$NON-NLS-1$
+            }
+            if (checkSum != null && checkSum.containsKey("md5")) { //$NON-NLS-1$
+                artifact.setMd5(checkSum.getString("md5")); //$NON-NLS-1$
+            }
+        }
+    }
+
     /*
      * (non-Javadoc)
      * 
@@ -216,7 +243,7 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
     @Override
     public void deploy(File content, String groupId, String artifactId, String classifier, String extension, String version)
             throws Exception {
-        String repositoryId = "";
+        String repositoryId = ""; //$NON-NLS-1$
         boolean isRelease = !version.endsWith(MavenUrlHelper.VERSION_SNAPSHOT);
         if (isRelease) {
             repositoryId = serverBean.getRepositoryId();
@@ -238,7 +265,7 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
     @Override
     public void deployWithPOM(File content, File pomFile, String groupId, String artifactId, String classifier, String extension,
             String version) throws Exception {
-        String repositoryId = "";
+        String repositoryId = ""; //$NON-NLS-1$
         boolean isRelease = !version.endsWith(MavenUrlHelper.VERSION_SNAPSHOT);
         if (isRelease) {
             repositoryId = serverBean.getRepositoryId();

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/Nexus2RepositoryHandler.java

@@ -137,5 +137,12 @@ public class Nexus2RepositoryHandler extends AbstractArtifactRepositoryHandler {
         RepositorySystemFactory.deployWithPOM(content, pomFile, localRepository, repositoryId, repositoryurl,
                 serverBean.getUserName(), serverBean.getPassword(), groupId, artifactId, classifier, extension, version);
     }
+    
+    @Override
+    public String resolveRemoteSha1(MavenArtifact artifact, boolean fromRelease) throws Exception {
+        return NexusServerUtils.resolveSha1(serverBean.getServer(), serverBean.getUserName(), serverBean.getPassword(),
+                fromRelease ? serverBean.getRepositoryId() : serverBean.getSnapshotRepId(), artifact.getGroupId(),
+                artifact.getArtifactId(), artifact.getVersion(), artifact.getType());
+    }
 
 }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/Nexus3RepositoryHandler.java

@@ -14,29 +14,35 @@ package org.talend.librariesmanager.nexus;
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.http.Header;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.ClientProtocolException;
-import org.apache.http.client.fluent.Request;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.entity.ContentType;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.message.BasicHeader;
-import org.apache.http.util.EntityUtils;
+import org.apache.http.params.CoreConnectionPNames;
+import org.apache.log4j.Logger;
 import org.eclipse.m2e.core.MavenPlugin;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.TalendProxySelector.IProxySelectorProvider;
+import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.nexus.HttpClientTransport;
 import org.talend.core.nexus.IRepositoryArtifactHandler;
+import org.talend.core.nexus.NexusServerUtils;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.designer.maven.aether.RepositorySystemFactory;
-
-import net.sf.json.JSONArray;
-import net.sf.json.JSONObject;
+import org.talend.librariesmanager.nexus.nexus3.handler.AbsNexus3SearchHandler;
+import org.talend.librariesmanager.nexus.nexus3.handler.INexus3SearchHandler;
+import org.talend.librariesmanager.nexus.nexus3.handler.Nexus3BetaSearchHandler;
+import org.talend.librariesmanager.nexus.nexus3.handler.Nexus3ScriptSearchHandler;
+import org.talend.librariesmanager.nexus.nexus3.handler.Nexus3V1SearchHandler;
 
 /**
  * created by wchen on Aug 2, 2017 Detailled comment
@@ -44,10 +50,14 @@ import net.sf.json.JSONObject;
  */
 public class Nexus3RepositoryHandler extends AbstractArtifactRepositoryHandler {
 
-    private String SEARCH_SERVICE = "service/rest/v1/script/search/run";
+    private static Logger LOGGER = Logger.getLogger(Nexus3RepositoryHandler.class);
 
     private String REP_PREFIX_PATH = "/repository/";
 
+    private ThreadLocal<INexus3SearchHandler> currentQueryHandlerCopy = new ThreadLocal<INexus3SearchHandler>();
+
+    private static final ConcurrentHashMap<ArtifactRepositoryBean, INexus3SearchHandler> LAST_HANDLER_MAP = new ConcurrentHashMap<ArtifactRepositoryBean, INexus3SearchHandler>();
+    
     @Override
     public IRepositoryArtifactHandler clone() {
         return new Nexus3RepositoryHandler();
@@ -92,11 +102,25 @@ public class Nexus3RepositoryHandler extends AbstractArtifactRepositoryHandler {
         HttpGet get = new HttpGet(repositoryUrl);
         get.addHeader(authority);
         DefaultHttpClient httpclient = new DefaultHttpClient();
-        HttpResponse response = httpclient.execute(get);
-        if (response.getStatusLine().getStatusCode() == 200) {
-            return true;
+        
+        httpclient.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, NexusServerUtils.getTimeout());
+        httpclient.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, NexusServerUtils.getTimeout());
+        IProxySelectorProvider proxySelector = null;
+        try {
+            try {
+                proxySelector = HttpClientTransport.addProxy(httpclient, new URI(repositoryUrl));
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+            HttpResponse response = httpclient.execute(get);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                return true;
+            }
+            return false;
+        } finally {
+            HttpClientTransport.removeProxy(proxySelector);
+            httpclient.getConnectionManager().shutdown();
         }
-        return false;
     }
 
     /*
@@ -108,75 +132,64 @@ public class Nexus3RepositoryHandler extends AbstractArtifactRepositoryHandler {
     @Override
     public List<MavenArtifact> search(String groupIdToSearch, String artifactId, String versionToSearch, boolean fromRelease,
             boolean fromSnapshot) throws Exception {
-        String serverUrl = serverBean.getServer();
-        if (!serverUrl.endsWith("/")) {
-            serverUrl = serverUrl + "/";
-        }
-        String searchUrl = serverUrl + SEARCH_SERVICE;
-        Request request = Request.Post(searchUrl);
-        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword();
-        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes()));
-        Header authority = new BasicHeader("Authorization", basicAuth);
-        Header contentType = new BasicHeader("Content-Type", "text/plain");
-        request.addHeader(contentType);
-        request.addHeader(authority);
+
         List<MavenArtifact> resultList = new ArrayList<MavenArtifact>();
         if (fromRelease) {
-            resultList.addAll(doSearch(request, serverBean.getRepositoryId(), groupIdToSearch, artifactId, versionToSearch));
+            resultList.addAll(doSearch(serverBean.getRepositoryId(), groupIdToSearch, artifactId, versionToSearch));
         }
         if (fromSnapshot) {
-            resultList.addAll(doSearch(request, serverBean.getSnapshotRepId(), groupIdToSearch, artifactId, versionToSearch));
+            resultList.addAll(doSearch(serverBean.getSnapshotRepId(), groupIdToSearch, artifactId, versionToSearch));
         }
 
         return resultList;
     }
 
-    private List<MavenArtifact> doSearch(Request request, String repositoryId, String groupIdToSearch, String artifactId,
-            String versionToSearch) throws Exception {
-        List<MavenArtifact> resultList = new ArrayList<MavenArtifact>();
-        JSONObject body = new JSONObject();
-        body.put("repositoryId", repositoryId);
-        if (groupIdToSearch != null) {
-            body.put("g", groupIdToSearch);
-        }
-        if (artifactId != null) {
-            body.put("a", artifactId);
-        }
-        if (versionToSearch != null) {
-            body.put("v", versionToSearch);
-        }
-        request.bodyString(body.toString(),
-                ContentType.create(ContentType.APPLICATION_JSON.getMimeType(), StandardCharsets.UTF_8));
-        HttpResponse response = request.execute().returnResponse();
-        String content = EntityUtils.toString(response.getEntity());
-        if (content.isEmpty()) {
-            return resultList;
-        }
-        JSONObject responseObject = new JSONObject().fromObject(content);
-        String resultStr = responseObject.getString("result");
-        JSONArray resultArray = null;
-        try {
-            resultArray = new JSONArray().fromObject(resultStr);
-        } catch (Exception e) {
-            throw new Exception(resultStr);
-        }
-        if (resultArray != null) {
-            for (int i = 0; i < resultArray.size(); i++) {
-                JSONObject jsonObject = resultArray.getJSONObject(i);
-                MavenArtifact artifact = new MavenArtifact();
-                artifact.setGroupId(jsonObject.getString("groupId"));
-                artifact.setArtifactId(jsonObject.getString("artifactId"));
-                artifact.setVersion(jsonObject.getString("version"));
-                artifact.setType(jsonObject.getString("extension"));
-                artifact.setDescription(jsonObject.getString("description"));
-                artifact.setLastUpdated(jsonObject.getString("last_updated"));
-                // artifact.setLicense(jsonObject.getString("license"));
-                // artifact.setLicenseUrl(jsonObject.getString("licenseUrl"));
-                // artifact.setUrl(jsonObject.getString("url"));
-                resultList.add(artifact);
+    private List<MavenArtifact> doSearch(String repositoryId, String groupIdToSearch, String artifactId, String versionToSearch)
+            throws Exception {
+        INexus3SearchHandler currentQueryHandler = currentQueryHandlerCopy.get();
+        if (currentQueryHandler == null) {
+            currentQueryHandler = LAST_HANDLER_MAP.get(serverBean);
+            if (currentQueryHandler == null) {
+                currentQueryHandler = createQueryHandler(0);
             }
         }
-        return resultList;
+        List<MavenArtifact> result = new ArrayList<MavenArtifact>();
+        try {
+            result = currentQueryHandler.search(repositoryId, groupIdToSearch, artifactId, versionToSearch);
+        } catch (Exception ex) {
+            for (int i = 0; i < 3; i++) {// Try to other version
+                INexus3SearchHandler handler = createQueryHandler(i);
+                if (handler != currentQueryHandler) {
+                    try {
+                        result = handler.search(repositoryId, groupIdToSearch, artifactId, versionToSearch);
+                        currentQueryHandler = handler;
+                        LOGGER.info(
+                                "Switch to new search handler,the handler version is:" + currentQueryHandler.getHandlerVersion());
+                        break;
+                    } catch (Exception e) {
+                        LOGGER.info("Try to switch search handler failed" + e.getMessage());
+                    }
+                }
+            }
+        }
+        currentQueryHandlerCopy.set(currentQueryHandler);
+        LAST_HANDLER_MAP.put(serverBean, currentQueryHandler);
+        return result;
+    }
+
+    private AbsNexus3SearchHandler createQueryHandler(int seq) {
+        AbsNexus3SearchHandler retHandler;
+        switch (seq) {
+        case 1:
+            retHandler = new Nexus3BetaSearchHandler(serverBean);
+            break;
+        case 2:
+            retHandler = new Nexus3ScriptSearchHandler(serverBean);
+            break;
+        default:
+            retHandler = new Nexus3V1SearchHandler(serverBean);
+        }
+        return retHandler;
     }
 
     /*

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/nexus3/handler/AbsNexus3SearchHandler.java

@@ -0,0 +1,248 @@
+package org.talend.librariesmanager.nexus.nexus3.handler;
+
+// ============================================================================
+//
+// Copyright (C) 2006-2018 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.util.EntityUtils;
+import org.apache.log4j.Logger;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.nexus.HttpClientTransport;
+import org.talend.core.runtime.maven.MavenArtifact;
+
+import net.sf.json.JSONArray;
+import net.sf.json.JSONObject;
+
+public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
+    private static Logger log = Logger.getLogger(AbsNexus3SearchHandler.class);
+    protected ArtifactRepositoryBean serverBean;
+    
+    /**
+     * {@value}
+     * <p>
+     * System property of nexus3 socket timeout, the unit is second.
+     */
+    private final String KEY_NEXUS3_SOCKET_TIMEOUT = "nexus3.socket.timeout";
+
+    private final int DEFAULT_SOCKET_TIMEOUT = 10 * 1000; // The default value is 10S
+
+    public AbsNexus3SearchHandler(ArtifactRepositoryBean serverBean) {
+        this.serverBean = serverBean;
+    }
+
+    protected String getServerUrl() {
+        String serverUrl = serverBean.getServer();
+        if (!serverUrl.endsWith("/")) { //$NON-NLS-1$
+            serverUrl = serverUrl + "/"; //$NON-NLS-1$
+        }
+        return serverUrl;
+    }
+
+    protected abstract String getSearchUrl();
+
+    @Override
+    public List<MavenArtifact> search(String repositoryId, String groupIdToSearch, String artifactId, String versionToSearch)
+            throws Exception {
+        List<MavenArtifact> resultList = new ArrayList<MavenArtifact>();
+        String searchUrl = getSearchUrl();
+        String continuationToken = null;
+        while (true) {
+            String query = getQueryParameter(repositoryId, groupIdToSearch, artifactId, versionToSearch, continuationToken);
+            String content = doRequest(searchUrl + query);
+            continuationToken = parseResult(content, resultList);
+            if (continuationToken == null) {
+                break;
+            }
+        }
+        return resultList;
+    }
+
+    protected String doRequest(final String url) throws URISyntaxException, Exception {
+        final StringBuffer sb = new StringBuffer();
+        new HttpClientTransport(url, serverBean.getUserName(), serverBean.getPassword()) {
+
+            @Override
+            protected HttpResponse execute(IProgressMonitor monitor, DefaultHttpClient httpClient, URI targetURI)
+                    throws Exception {
+                HttpGet httpGet = new HttpGet(targetURI);
+                String userName = serverBean.getUserName();
+                if (StringUtils.isNotBlank(userName) && !Boolean.getBoolean("talend.studio.search.nexus3.disableBasicAuth")) {
+                    String auth = userName + ":" + serverBean.getPassword();
+                    String authHeader = "Basic " + new String(Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)));
+                    httpGet.setHeader(HttpHeaders.AUTHORIZATION, authHeader);
+                    httpClient.setCredentialsProvider(null);
+                }
+                HttpResponse response = httpClient.execute(httpGet);
+                if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+                    sb.append(EntityUtils.toString(response.getEntity()));
+                } else {
+                    throw new Exception(response.toString());
+                }
+                return response;
+            }
+
+        }.doRequest(null, new URI(url));
+        return sb.toString();
+    }
+
+    protected String parseResult(String content, List<MavenArtifact> resultList) throws Exception {
+        if (StringUtils.isEmpty(content)) {
+            return null;
+        }
+        JSONObject responseObject = JSONObject.fromObject(content);
+        String resultStr = responseObject.getString("items"); //$NON-NLS-1$
+        String continuationToken = responseObject.getString("continuationToken");
+        if (StringUtils.isEmpty(continuationToken) || "null".equalsIgnoreCase(continuationToken)) {
+            continuationToken = null;
+        }
+        JSONArray resultArray = null;
+        try {
+            resultArray = JSONArray.fromObject(resultStr);
+        } catch (Exception e) {
+            throw new Exception(resultStr);
+        }
+        if (resultArray != null) {
+            for (int i = 0; i < resultArray.size(); i++) {
+                JSONObject jsonObject = resultArray.getJSONObject(i);
+                MavenArtifact artifact = new MavenArtifact();
+                artifact.setGroupId(jsonObject.getString("group")); //$NON-NLS-1$
+                artifact.setArtifactId(jsonObject.getString("name")); //$NON-NLS-1$
+                artifact.setVersion(jsonObject.getString("version")); //$NON-NLS-1$
+                JSONArray assertsArray = jsonObject.getJSONArray("assets"); //$NON-NLS-1$                
+                artifact.setType(getPackageType(assertsArray));
+                if (artifact.getType() != null) {
+                    fillCheckSumData(assertsArray, artifact);
+                    resultList.add(artifact);
+                }
+
+            }
+        }
+
+        return continuationToken;
+    }
+
+    protected String getPackageType(JSONArray assertsArray) {
+        String type = null;
+        if (assertsArray != null) {
+            for (int i = 0; i < assertsArray.size(); i++) {
+                JSONObject jsonObject = assertsArray.getJSONObject(i);
+                String path = jsonObject.getString("path"); //$NON-NLS-1$
+                if (path != null && path.endsWith(".exe")) { //$NON-NLS-1$
+                    return "exe"; //$NON-NLS-1$
+                }
+                if (path != null && path.endsWith(".zip")) { //$NON-NLS-1$
+                    return "zip"; //$NON-NLS-1$
+                }
+                if (path != null && path.endsWith(".jar")) { //$NON-NLS-1$
+                    return "jar"; //$NON-NLS-1$
+                }
+                if (path != null && path.endsWith(".pom")) { //$NON-NLS-1$
+                    type = "pom"; //$NON-NLS-1$
+                }
+            }
+        }
+        return type;
+    }
+
+    private void fillCheckSumData(JSONArray assertsArray, MavenArtifact artifact) {
+        if (assertsArray != null) {
+            for (int i = 0; i < assertsArray.size(); i++) {
+                JSONObject jsonObject = assertsArray.getJSONObject(i);
+                if (jsonObject.containsKey("path")) { //$NON-NLS-1$
+                    String path = jsonObject.getString("path"); //$NON-NLS-1$
+                    if (path != null && path.endsWith(artifact.getType())) { // $NON-NLS-1$
+                        if (jsonObject.containsKey("checksum")) {
+                            JSONObject checksumObject = jsonObject.getJSONObject("checksum"); //$NON-NLS-1$
+                            if (checksumObject != null && checksumObject.containsKey("sha1")) {//$NON-NLS-1$
+                                artifact.setSha1(checksumObject.getString("sha1")); //$NON-NLS-1$
+                                artifact.setMd5(checksumObject.getString("md5")); //$NON-NLS-1$
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    protected String getQueryParameter(String repositoryId, String groupIdToSearch, String artifactId, String versionToSearch,
+            String continuationToken) {
+        StringBuffer sb = new StringBuffer();
+        boolean hasParameter = false;
+        if (StringUtils.isNoneEmpty(repositoryId)) {
+            sb.append("repository=").append(repositoryId); //$NON-NLS-1$
+            hasParameter = true;
+        }
+        if (StringUtils.isNoneEmpty(groupIdToSearch)) {
+            if (hasParameter) {
+                sb.append("&"); //$NON-NLS-1$
+            }
+            sb.append("group=").append(groupIdToSearch); //$NON-NLS-1$
+            hasParameter = true;
+        }
+        if (StringUtils.isNoneEmpty(artifactId)) {
+            if (hasParameter) {
+                sb.append("&"); //$NON-NLS-1$
+            }
+            sb.append("name=").append(artifactId); //$NON-NLS-1$
+            hasParameter = true;
+        }
+        if (StringUtils.isNoneEmpty(versionToSearch)) {
+            if (hasParameter) {
+                sb.append("&"); //$NON-NLS-1$
+            }
+            sb.append("version=").append(versionToSearch); //$NON-NLS-1$
+            hasParameter = true;
+        }
+        if (StringUtils.isNoneEmpty(continuationToken)) {
+            if (hasParameter) {
+                sb.append("&"); //$NON-NLS-1$
+            }
+            sb.append("continuationToken=").append(continuationToken); //$NON-NLS-1$
+            hasParameter = true;
+        }
+        return sb.toString();
+    }
+
+    protected String getAuthenticationItem() {
+        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword(); //$NON-NLS-1$
+        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes())); //$NON-NLS-1$
+        return basicAuth;
+    }
+    
+    protected int getNexus3SocketTimeout() {
+        int socketTimeout = DEFAULT_SOCKET_TIMEOUT;
+        String strValue = System.getProperty(KEY_NEXUS3_SOCKET_TIMEOUT);
+        if (StringUtils.isNotEmpty(strValue)) {
+            try {
+                int value = Integer.parseInt(strValue);
+                socketTimeout = value * 1000;
+            } catch (NumberFormatException ex) {
+                log.error("Parse nexus3 socket timeout error:" + ex.getMessage());
+            }
+        }
+        return socketTimeout;
+    }
+}

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/nexus3/handler/INexus3SearchHandler.java

@@ -0,0 +1,26 @@
+package org.talend.librariesmanager.nexus.nexus3.handler;
+
+// ============================================================================
+//
+// Copyright (C) 2006-2018 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+import java.util.List;
+
+import org.talend.core.runtime.maven.MavenArtifact;
+
+public interface INexus3SearchHandler {
+
+    public List<MavenArtifact> search(String repositoryId, String groupIdToSearch, String artifactId, String versionToSearch)
+            throws Exception;
+
+    public String getHandlerVersion();
+
+}

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/nexus3/handler/Nexus3BetaSearchHandler.java

@@ -0,0 +1,20 @@
+package org.talend.librariesmanager.nexus.nexus3.handler;
+
+import org.talend.core.nexus.ArtifactRepositoryBean;
+
+public class Nexus3BetaSearchHandler extends AbsNexus3SearchHandler {
+
+    private String SEARCH_SERVICE = "service/rest/beta/search?"; //$NON-NLS-1$
+
+    public Nexus3BetaSearchHandler(ArtifactRepositoryBean serverBean) {
+        super(serverBean);
+    }
+
+    protected String getSearchUrl() {
+        return this.getServerUrl() + SEARCH_SERVICE;
+    }
+
+    public String getHandlerVersion() {
+        return "Nexus3.beta"; //$NON-NLS-1$
+    }
+}

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/nexus3/handler/Nexus3ScriptSearchHandler.java

@@ -0,0 +1,110 @@
+package org.talend.librariesmanager.nexus.nexus3.handler;
+
+// ============================================================================
+//
+// Copyright (C) 2006-2018 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.http.Header;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.fluent.Request;
+import org.apache.http.entity.ContentType;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.util.EntityUtils;
+import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.runtime.maven.MavenArtifact;
+
+import net.sf.json.JSONArray;
+import net.sf.json.JSONObject;
+
+public class Nexus3ScriptSearchHandler extends AbsNexus3SearchHandler {
+
+    private String SEARCH_SERVICE = "service/rest/v1/script/search/run"; //$NON-NLS-1$
+
+    public Nexus3ScriptSearchHandler(ArtifactRepositoryBean serverBean) {
+        super(serverBean);
+    }
+
+    @Override
+    public List<MavenArtifact> search(String repositoryId, String groupIdToSearch, String artifactId, String versionToSearch)
+            throws Exception {
+        String searchUrl = getSearchUrl();
+        Request request = this.getRequest(searchUrl);
+        List<MavenArtifact> resultList = new ArrayList<MavenArtifact>();
+        JSONObject body = new JSONObject();
+        body.put("repositoryId", repositoryId); //$NON-NLS-1$
+        if (groupIdToSearch != null) {
+            body.put("g", groupIdToSearch); //$NON-NLS-1$
+        }
+        if (artifactId != null) {
+            body.put("a", artifactId); //$NON-NLS-1$
+        }
+        if (versionToSearch != null) {
+            body.put("v", versionToSearch); //$NON-NLS-1$
+        }
+        request.bodyString(body.toString(),
+                ContentType.create(ContentType.APPLICATION_JSON.getMimeType(), StandardCharsets.UTF_8));
+        HttpResponse response = request.execute().returnResponse();
+        if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+            String content = EntityUtils.toString(response.getEntity());
+            if (content.isEmpty()) {
+                return resultList;
+            }
+            JSONObject responseObject = JSONObject.fromObject(content);
+            String resultStr = responseObject.getString("result"); //$NON-NLS-1$
+            JSONArray resultArray = null;
+            try {
+                resultArray = JSONArray.fromObject(resultStr);
+            } catch (Exception e) {
+                throw new Exception(resultStr);
+            }
+            if (resultArray != null) {
+                for (int i = 0; i < resultArray.size(); i++) {
+                    JSONObject jsonObject = resultArray.getJSONObject(i);
+                    MavenArtifact artifact = new MavenArtifact();
+                    artifact.setGroupId(jsonObject.getString("groupId")); //$NON-NLS-1$
+                    artifact.setArtifactId(jsonObject.getString("artifactId")); //$NON-NLS-1$
+                    artifact.setVersion(jsonObject.getString("version")); //$NON-NLS-1$
+                    artifact.setType(jsonObject.getString("extension")); //$NON-NLS-1$
+                    artifact.setDescription(jsonObject.getString("description")); //$NON-NLS-1$
+                    artifact.setLastUpdated(jsonObject.getString("last_updated")); //$NON-NLS-1$
+                    resultList.add(artifact);
+                }
+            }
+            return resultList;
+        } else {
+            throw new Exception(response.toString());
+        }
+    }
+
+    protected Request getRequest(String searchUrl) {
+        Request request = Request.Post(searchUrl);
+        Header authority = new BasicHeader("Authorization", getAuthenticationItem()); //$NON-NLS-1$
+        Header contentType = new BasicHeader("Content-Type", "text/plain"); //$NON-NLS-1$ //$NON-NLS-2$
+        request.addHeader(contentType);
+        request.addHeader(authority);
+        request.socketTimeout(this.getNexus3SocketTimeout());
+        return request;
+    }
+
+    protected String getSearchUrl() {
+        return this.getServerUrl() + SEARCH_SERVICE;
+    }
+
+    public String getHandlerVersion() {
+        return "Nexus3.script"; //$NON-NLS-1$
+    }
+
+}

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/nexus3/handler/Nexus3V1SearchHandler.java

@@ -0,0 +1,20 @@
+package org.talend.librariesmanager.nexus.nexus3.handler;
+
+import org.talend.core.nexus.ArtifactRepositoryBean;
+
+public class Nexus3V1SearchHandler extends AbsNexus3SearchHandler {
+
+    private String SEARCH_SERVICE = "service/rest/v1/search?";
+
+    public Nexus3V1SearchHandler(ArtifactRepositoryBean serverBean) {
+        super(serverBean);
+    }
+
+    protected String getSearchUrl() {
+        return this.getServerUrl() + SEARCH_SERVICE;
+    }
+
+    public String getHandlerVersion() {
+        return "Nexus3.V1"; //$NON-NLS-1$
+    }
+}

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/utils/ShareLibrariesUtil.java

@@ -0,0 +1,132 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.librariesmanager.nexus.utils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.codehaus.jackson.map.util.ISO8601Utils;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.nexus.IRepositoryArtifactHandler;
+import org.talend.core.runtime.maven.MavenArtifact;
+
+public class ShareLibrariesUtil {
+
+    public static boolean isSameFileWithRemote(File localFile, List<MavenArtifact> artifactList,
+            ArtifactRepositoryBean customNexusServer, IRepositoryArtifactHandler customerRepHandler, boolean isSnapshotVersion)
+            throws Exception {
+        String localFileShaCode = "";
+
+        try (FileInputStream fi = new FileInputStream(localFile)) {
+
+            localFileShaCode = DigestUtils.shaHex(fi);
+
+        }
+        String remoteSha1 = null;
+        if (ArtifactRepositoryBean.NexusType.ARTIFACTORY.name().equalsIgnoreCase(customNexusServer.getType())) {
+            MavenArtifact lastUpdatedArtifact = getLateUpdatedMavenArtifact(artifactList);
+            if (lastUpdatedArtifact != null) {
+                remoteSha1 = lastUpdatedArtifact.getSha1();
+            }
+        } else if (ArtifactRepositoryBean.NexusType.NEXUS_3.name().equalsIgnoreCase(customNexusServer.getType())) {
+            MavenArtifact lastUpdatedArtifact = artifactList.stream().max(Comparator.comparing(e -> e.getVersion())).get();
+            if (lastUpdatedArtifact != null) {
+                remoteSha1 = lastUpdatedArtifact.getSha1();
+            }
+        } else {
+            if (!isSnapshotVersion && !Boolean.getBoolean("force_libs_release_update")) {
+                return true;
+            }
+            MavenArtifact lastUpdatedArtifact = artifactList.get(0);
+            if (lastUpdatedArtifact != null) {
+                remoteSha1 = customerRepHandler.resolveRemoteSha1(lastUpdatedArtifact, !isSnapshotVersion);
+            }
+        }
+        if (StringUtils.equals(localFileShaCode, remoteSha1)) {
+            return true;
+        }
+        return false;
+    }
+
+    private static MavenArtifact getLateUpdatedMavenArtifact(List<MavenArtifact> artifactList) {
+        if (artifactList.size() == 1) {
+            return artifactList.get(0);
+        }
+        MavenArtifact latestVersion = null;
+        Date lastUpdate = null;
+        for (MavenArtifact art : artifactList) {
+            if (latestVersion == null) {
+                if (art.getLastUpdated() != null) {
+                    latestVersion = art;
+                    lastUpdate = parsetDate(art.getLastUpdated());
+                }
+            } else if (art.getLastUpdated() != null && lastUpdate != null) {
+                Date artLastUpdate = parsetDate(art.getLastUpdated());
+                if (artLastUpdate != null && lastUpdate.getTime() < artLastUpdate.getTime()) {
+                    latestVersion = art;
+                    lastUpdate = artLastUpdate;
+                }
+            }
+        }
+        if (latestVersion != null) {
+            return latestVersion;
+        } else {
+            return artifactList.get(artifactList.size() - 1);
+        }
+    }
+
+    private static Date parsetDate(String strDate) {
+        Date date = null;
+        if (strDate != null) {
+            try {
+                date = ISO8601Utils.parse(strDate);
+            } catch (Exception ex) {
+                ExceptionHandler.process(ex);
+            }
+        }
+        return date;
+    }
+
+    public static void putArtifactToMap(MavenArtifact artifact, Map<String, List<MavenArtifact>> map, boolean isShapshot) {
+        String key = getArtifactKey(artifact, isShapshot);
+        List<MavenArtifact> list = map.get(key);
+        if (list == null) {
+            list = new ArrayList<MavenArtifact>();
+            map.put(key, list);
+        }
+        list.add(artifact);
+    }
+
+    public static String getArtifactKey(MavenArtifact artifact, boolean isShapshot) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(artifact.getGroupId()).append("-");
+        sb.append(artifact.getArtifactId()).append("-");
+        String version = artifact.getVersion();
+        if (isShapshot) {
+            version = VersionUtil.getSNAPSHOTVersion(version);
+        }
+        sb.append(version);
+        if (StringUtils.isNotEmpty(artifact.getClassifier())) {
+            sb.append("-").append(artifact.getClassifier());
+        }
+        return sb.toString();
+    }
+}

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/utils/VersionUtil.java

@@ -0,0 +1,32 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.librariesmanager.nexus.utils;
+
+import org.talend.core.runtime.maven.MavenUrlHelper;
+
+/**
+ * DOC hwang  class global comment. Detailled comment
+ */
+public class VersionUtil {
+    
+    public static String getSNAPSHOTVersion(String rVersion) {
+        if(rVersion == null) {
+            return rVersion;
+        }
+        if(rVersion.contains("-")) {
+            return rVersion.substring(0, rVersion.indexOf("-") + 1) + MavenUrlHelper.VERSION_SNAPSHOT;
+        }
+        return rVersion;
+    }
+
+}

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/prefs/LibrariesManagerUtils.java

@@ -18,11 +18,16 @@ import java.util.List;
 import java.util.Set;
 
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.preferences.IEclipsePreferences;
+import org.eclipse.core.runtime.preferences.InstanceScope;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.NetworkUtil;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.language.ECodeLanguage;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
 import org.talend.core.model.process.INode;
+import org.talend.core.prefs.ITalendCorePrefConstants;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.librariesmanager.model.ModulesNeededProvider;
 
@@ -103,4 +108,15 @@ public class LibrariesManagerUtils {
         }
         return updatedModules;
     }
+
+    public static boolean shareLibsAtStartup() {
+        boolean ret = ITalendCorePrefConstants.NEXUS_SHARE_LIBS_DEFAULT;
+        try {
+            IEclipsePreferences node = InstanceScope.INSTANCE.getNode(NetworkUtil.ORG_TALEND_DESIGNER_CORE);
+            ret = node.getBoolean(ITalendCorePrefConstants.NEXUS_SHARE_LIBS, ITalendCorePrefConstants.NEXUS_SHARE_LIBS_DEFAULT);
+        } catch (Throwable e) {
+            ExceptionHandler.process(e);
+        }
+        return ret;
+    }
 }

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/utils/DBConnectionContextUtils.java

@@ -1321,34 +1321,21 @@ public final class DBConnectionContextUtils {
             }
         }
 
-        // Added 20130311 TDQ-7000, when it is context mode and not general jdbc, reset the url.
-        if (contextType != null
-                && !EDatabaseTypeName.GENERAL_JDBC
-                        .equals(EDatabaseTypeName.getTypeFromDbType(dbConn.getDatabaseType()))) {
-            String newURL = null;
-            if (EDatabaseTypeName.IMPALA.equals(EDatabaseTypeName.getTypeFromDbType(dbConn.getDatabaseType()))) {
-                newURL =
-                        DatabaseConnStrUtil.getImpalaString(cloneConn, cloneConn.getServerName(), cloneConn.getPort(),
-                                cloneConn.getSID(), DbConnStrForHive.URL_HIVE_2_TEMPLATE);
-            } else {
-                newURL =
-                        DatabaseConnStrUtil.getURLString(cloneConn.getDatabaseType(), dbConn.getDbVersionString(),
-                                server, username, password, port, sidOrDatabase, filePath.toLowerCase(), datasource,
-                                dbRootPath, additionParam);
-            }
-            cloneConn.setURL(newURL);
-            return cloneConn;
-        }// ~
-
-        if (dbConn.getURL() != null && !dbConn.getURL().equals("") && EDatabaseTypeName.GENERAL_JDBC
-                .equals(EDatabaseTypeName.getTypeFromDbType(dbConn.getDatabaseType()))) { //$NON-NLS-1$
+        if (dbConn.getURL() != null && !dbConn.getURL().equals("") //$NON-NLS-1$
+                && EDatabaseTypeName.GENERAL_JDBC.equals(EDatabaseTypeName.getTypeFromDbType(dbConn.getDatabaseType()))) {
             cloneConn.setURL(url);
             return cloneConn;
         }
-        String newURL =
-                DatabaseConnStrUtil.getURLString(cloneConn.getDatabaseType(), dbConn.getDbVersionString(), server,
-                        username, password, port, sidOrDatabase, filePath.toLowerCase(), datasource, dbRootPath,
-                        additionParam);
+
+        // Added 20130311 TDQ-7000, when it is context mode and not general jdbc, reset the url.
+        String newURL = null;
+        if (EDatabaseTypeName.IMPALA.equals(EDatabaseTypeName.getTypeFromDbType(dbConn.getDatabaseType()))) {
+            newURL = DatabaseConnStrUtil.getImpalaString(cloneConn, cloneConn.getServerName(), cloneConn.getPort(),
+                    cloneConn.getSID(), DbConnStrForHive.URL_HIVE_2_TEMPLATE);
+        } else {
+            newURL = DatabaseConnStrUtil.getURLString(cloneConn.getDatabaseType(), dbConn.getDbVersionString(), server, username,
+                    password, port, sidOrDatabase, filePath.toLowerCase(), datasource, dbRootPath, additionParam);
+        }
         cloneConn.setURL(newURL);
         return cloneConn;
     }

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/utils/XsdMetadataUtils.java

@@ -218,6 +218,12 @@ public final class XsdMetadataUtils {
                     break;
                 }
             }
+
+            if (node == null) {
+                // no declaration in the present XSD
+                return;
+            }
+
             node = populationUtil.getSchemaTree(xsdSchema, node);
             orderId = 1;
             loopElementFound = false;