fix issues

found dialog when project is in MERGING state
https://jira.talendforge.org/browse/TUP-27003

main/plugins/org.talend.common.ui.runtime/src/main/java/org/talend/commons/ui/runtime/exception/ExceptionMessageDialog.java

@@ -21,6 +21,7 @@ import org.eclipse.swt.SWT;
 import org.eclipse.swt.graphics.Image;
 import org.eclipse.swt.graphics.Point;
 import org.eclipse.swt.layout.GridData;
+import org.eclipse.swt.widgets.Button;
 import org.eclipse.swt.widgets.Composite;
 import org.eclipse.swt.widgets.Control;
 import org.eclipse.swt.widgets.Shell;
@@ -38,6 +39,8 @@ public class ExceptionMessageDialog extends MessageDialog {
 
     private String exceptionString = null;
 
+    private int[] diabledButtonIndex = new int[] {};
+
     public ExceptionMessageDialog(Shell parentShell, String dialogTitle, Image dialogTitleImage, String dialogMessage,
             int dialogImageType, String[] dialogButtonLabels, int defaultIndex, Throwable ex) {
         super(parentShell, dialogTitle, dialogTitleImage, dialogMessage, dialogImageType, dialogButtonLabels, defaultIndex);
@@ -147,4 +150,21 @@ public class ExceptionMessageDialog extends MessageDialog {
         this.exceptionString = exceptionString;
     }
 
+    public void setDisabledButtons(int[] index) {
+        this.diabledButtonIndex = index;
+    }
+
+    @Override
+    protected void createButtonsForButtonBar(Composite parent) {
+        super.createButtonsForButtonBar(parent);
+        if (diabledButtonIndex == null) {
+            return;
+        }
+        for (int index : diabledButtonIndex) {
+            Button button = super.getButton(index);
+            button.setEnabled(false);
+        }
+    }
+
+
 }

main/plugins/org.talend.commons.runtime/META-INF/MANIFEST.MF

@@ -13,7 +13,8 @@ Require-Bundle: org.apache.log4j;visibility:=reexport,
  org.talend.libraries.apache,
  com.ibm.icu,
  org.apache.commons.io,
- org.talend.utils
+ org.talend.utils,
+ org.eclipse.core.net
 Export-Package: org.talend.commons,
  org.talend.commons.exception,
  org.talend.commons.i18n,

main/plugins/org.talend.commons.runtime/src/org/talend/commons/CommonsPlugin.java

@@ -17,6 +17,7 @@ import java.io.InputStream;
 import java.net.URL;
 
 import org.apache.commons.lang.ArrayUtils;
+import org.eclipse.core.net.proxy.IProxyService;
 import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.preferences.InstanceScope;
@@ -24,6 +25,7 @@ import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleActivator;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.ServiceReference;
+import org.osgi.util.tracker.ServiceTracker;
 import org.talend.commons.exception.ExceptionService;
 import org.talend.commons.runtime.debug.TalendDebugHandler;
 import org.talend.commons.utils.system.EclipseCommandLine;
@@ -55,6 +57,8 @@ public class CommonsPlugin implements BundleActivator {
     // TESB-17856: For commandline builds ESB Micorservice bundle
     private static boolean isESBMicorservice = false;
 
+    private static ServiceTracker proxyTracker;
+
     public static boolean isWorkbenchCreated() {
         return isWorkbenchCreated;
     }
@@ -176,4 +180,13 @@ public class CommonsPlugin implements BundleActivator {
         return FileLocator.toFileURL(entry);
     }
 
+    public static IProxyService getProxyService() {
+        if (proxyTracker == null) {
+            proxyTracker = new ServiceTracker(Platform.getBundle(PLUGIN_ID).getBundleContext(), IProxyService.class.getName(),
+                    null);
+            proxyTracker.open();
+        }
+        return (IProxyService) proxyTracker.getService();
+    }
+
 }

main/plugins/org.talend.commons.runtime/src/org/talend/commons/i18n/internal/messages.properties

@@ -58,3 +58,6 @@ AS400ResultSet.unknowCloumn=Invalid argument\: unknown column name
 AS400ResultSet.parameterIndex=Invalid argument\: parameter index
 AS400ResultSet.outofRange=\ is out of range.
 ITaCoKitService.exception.multipleInstance=More than one instance found: {0}
+TalendProxySelector.exception.badUriMap=Bad uri map: {0}
+TalendProxySelector.exception.proxySelectionError=Error occurs when selecting proxy for {0}
+

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/database/Sybase16SADatabaseMetaData.java

@@ -0,0 +1,121 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.commons.utils.database;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+
+
+/**
+ * created by qiongli on 2013-11-13 Detailled comment
+ *
+ */
+public class Sybase16SADatabaseMetaData extends SybaseDatabaseMetaData {
+
+    private static Logger log = Logger.getLogger(Sybase16SADatabaseMetaData.class);
+
+    /**
+     * DOC qiongli SybaseIQDatabaseMetaData constructor comment.
+     *
+     * @param connection
+     * @throws SQLException
+     */
+    public Sybase16SADatabaseMetaData(Connection connection) throws SQLException {
+        super(connection);
+    }
+
+    public ResultSet getCatalogs(String login, String database) throws SQLException {
+        List<String[]> list = new ArrayList<String[]>();
+
+        List<String> catList = new ArrayList<String>();
+        if (!StringUtils.isEmpty(database)) {
+            catList.add(database);
+        }
+        
+        for (String catalogName : catList) {
+            String sql = createSqlByLoginAndCatalog(login, catalogName);
+            ResultSet rs = null;
+            Statement stmt = null;
+            try {
+                stmt = connection.createStatement();
+                rs = stmt.executeQuery(sql);
+
+                while (rs.next()) {
+                    int temp = rs.getInt(1);
+                    if (temp > 0) {
+                        String[] r = new String[] { catalogName };
+                        list.add(r);
+                    }
+                }
+            } catch (SQLException e) {
+                log.error(e);
+            } finally {
+                try {
+                    if (rs != null) {
+                        rs.close();
+                    }
+                    if (stmt != null) {
+                        stmt.close();
+                    }
+                } catch (Exception e) {
+                    log.error(e);
+                }
+            }
+        }
+
+        SybaseResultSet tableResultSet = new SybaseResultSet();
+        tableResultSet.setMetadata(new String[] { "TABLE_CAT" }); //$NON-NLS-1$
+        tableResultSet.setData(list);
+        return tableResultSet;
+    }
+    
+    @Override
+    public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern)
+            throws SQLException {
+        return super.getColumns(null, schemaPattern, tableNamePattern, columnNamePattern);
+    }
+
+    @Override
+    public ResultSet getPrimaryKeys(String catalog, String schema, String table) throws SQLException {
+        return super.getPrimaryKeys(null, schema, table);
+    }
+
+    @Override
+    public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types)
+            throws SQLException {
+        return super.getTables(null, schemaPattern, tableNamePattern, types);
+    }
+
+    /**
+     *
+     * get a sql query by login name and catalog name.
+     *
+     * @param loginName
+     * @param catalogName
+     * @return
+     */
+    protected String createSqlByLoginAndCatalog(String loginName, String catalogName) {
+        String sql = "select count(*) from " + catalogName
+                + ".dbo.sysusers where suid in (select suid from "+catalogName+".dbo.syslogins where name = '" + loginName
+                + "')";
+        return sql;
+    }
+
+}

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/network/TalendProxySelector.java

@@ -13,37 +13,211 @@
 package org.talend.commons.utils.network;
 
 import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.net.PasswordAuthentication;
 import java.net.Proxy;
 import java.net.ProxySelector;
 import java.net.SocketAddress;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Priority;
+import org.eclipse.core.internal.net.ProxyManager;
+import org.eclipse.core.net.proxy.IProxyService;
+import org.talend.commons.CommonsPlugin;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.i18n.internal.Messages;
+import org.talend.daikon.sandbox.properties.ClassLoaderIsolatedSystemProperties;
+
+import sun.net.spi.DefaultProxySelector;
 
 /**
  * DOC cmeng  class global comment. Detailled comment
  */
 public class TalendProxySelector extends ProxySelector {
 
+    public static final String PROP_PRINT_LOGS = "talend.studio.proxy.printLogs";
+
     private static final String ECLIPSE_PROXY_SELECTOR = ".EclipseProxySelector"; //$NON-NLS-1$
 
-    private ProxySelector defaultSelector;
+    private static final String PROP_ALLOW_PROXY_REDIRECT = "talend.studio.proxy.allowProxyRedirect";
 
-    final private List<IProxySelectorProvider> selectorProviders;
+    private static final String PROP_ALLOW_PROXY_REDIRECT_EXCLUDE = "talend.studio.proxy.redirect.whiteList";
+
+    private static final String PROP_EXECUTE_CONNECTION_FAILED = "talend.studio.proxy.executeConnectionFailed";
+
+    private static final String PROP_UPDATE_SYSTEM_PROPERTIES_FOR_JRE = "talend.studio.proxy.jre.updateSystemProperties";
+
+    private static final String PROP_CHECK_PROXY = "talend.studio.proxy.checkProxy";
+
+    private static final String PROP_VALIDATE_URI = "talend.studio.proxy.validateUri";
+
+    private static final String PROP_PROXY_SELECTOR = "talend.studio.proxy.selector";
+
+    private static final String PROP_PROXY_SELECTOR_DEFAULT = "default";
+
+    private static final String PROP_PROXY_SELECTOR_JRE = "jre";
+
+    private static final String PROP_PROXY_HOST_MAP = "talend.studio.proxy.hostMap";
+
+    private static final String PROP_DISABLE_DEFAULT_SELECTOR_PROVIDER = "talend.studio.proxy.disableDefaultSelectorProvider";
+
+    /**
+     * Example: update.talend.com,socket:http,https:http;nexus.talend.com,socket,http;,socket:http
+     */
+    private static final String PROP_PROXY_MAP_HOST_DEFAULT = "";
+
+    /**
+     * Example: svn.company.com;nexus.company.com
+     */
+    private static final String PROP_ALLOW_PROXY_REDIRECT_EXCLUDE_DEFAULT = "";
+
+    private static final String KEY_DEFAULT = ":default:";
+
+    private static Field uriHostField;
+
+    private static Method proxyManagerUpdateSystemPropertiesFunc;
+
+    private static boolean checkProxy = Boolean.valueOf(System.getProperty(PROP_CHECK_PROXY, Boolean.TRUE.toString()));
+
+    /**
+     * Note: eclipse default selector may be different between TOS and TIS, TOS may use jre one, TIS may use egit one
+     */
+    private ProxySelector eclipseDefaultSelector;
+
+    private ProxySelector jreDefaultSelector;
+
+    private EProxySelector eProxySelector;
+
+    final private Map<Object, Collection<IProxySelectorProvider>> selectorProviders;
+
+    private Map<String, Map<String, String>> hostMap;
+
+    private Set<String> redirectWhiteList;
 
     private volatile static TalendProxySelector instance;
 
     private static Object instanceLock = new Object();
 
-    private TalendProxySelector(final ProxySelector defaultSelector) {
-        this.defaultSelector = defaultSelector;
-        selectorProviders = new ArrayList<>();
+    private boolean printProxyLog = false;
+
+    private boolean allowProxyRedirect = false;
+
+    private boolean disableDefaultSelectorProvider = false;
+
+    private boolean validateUri = true;
+
+    private boolean executeConnectionFailed = true;
+
+    private boolean updateSystemPropertiesForJre = true;
+
+    private TalendProxySelector(final ProxySelector eclipseDefaultSelector) {
+        this.eclipseDefaultSelector = eclipseDefaultSelector;
+        this.jreDefaultSelector = new DefaultProxySelector();
+
+        selectorProviders = Collections.synchronizedMap(new HashMap<>());
+        allowProxyRedirect = Boolean.valueOf(System.getProperty(PROP_ALLOW_PROXY_REDIRECT, Boolean.FALSE.toString()));
+        disableDefaultSelectorProvider = Boolean
+                .valueOf(System.getProperty(PROP_DISABLE_DEFAULT_SELECTOR_PROVIDER, Boolean.FALSE.toString()));
+        printProxyLog = Boolean.valueOf(System.getProperty(PROP_PRINT_LOGS, Boolean.FALSE.toString()));
+        validateUri = Boolean.valueOf(System.getProperty(PROP_VALIDATE_URI, Boolean.TRUE.toString()));
+        executeConnectionFailed = Boolean.valueOf(System.getProperty(PROP_EXECUTE_CONNECTION_FAILED, Boolean.TRUE.toString()));
+        updateSystemPropertiesForJre = Boolean
+                .valueOf(System.getProperty(PROP_UPDATE_SYSTEM_PROPERTIES_FOR_JRE, Boolean.TRUE.toString()));
+
+        switch (System.getProperty(PROP_PROXY_SELECTOR, PROP_PROXY_SELECTOR_DEFAULT).toLowerCase()) {
+        case PROP_PROXY_SELECTOR_JRE:
+            this.eProxySelector = EProxySelector.jre;
+            break;
+        default:
+            this.eProxySelector = EProxySelector.eclipse_default;
+            break;
+        }
+
+        initHostMap();
+        initRedirectList();
+    }
+
+    private void initHostMap() {
+        try {
+            hostMap = new HashMap<>();
+            String property = System.getProperty(PROP_PROXY_HOST_MAP, PROP_PROXY_MAP_HOST_DEFAULT);
+            if (StringUtils.isEmpty(property)) {
+                return;
+            }
+            String[] splits = property.split(";");
+            for (String split : splits) {
+                try {
+                    int index = split.indexOf(',');
+                    String uri = split.substring(0, index);
+                    String key = StringUtils.strip(uri);
+                    if (StringUtils.isBlank(key)) {
+                        key = KEY_DEFAULT;
+                    }
+                    key = key.toLowerCase();
+                    Map<String, String> protocolMap = hostMap.get(key);
+                    if (protocolMap == null) {
+                        protocolMap = new HashMap<>();
+                        hostMap.put(key, protocolMap);
+                    }
+                    int protocolMapIndex = index + 1;
+                    String protocolMapStr = split.substring(protocolMapIndex);
+                    String[] entry = protocolMapStr.split(",");
+                    for (String pMap : entry) {
+                        try {
+                            String[] mapEntry = pMap.split(":");
+                            if (mapEntry.length != 2) {
+                                ExceptionHandler.process(
+                                        new Exception(Messages.getString("TalendProxySelector.exception.badUriMap", pMap)));
+                                continue;
+                            }
+                            protocolMap.put(mapEntry[0].toLowerCase(), mapEntry[1].toLowerCase());
+                        } catch (Exception e) {
+                            ExceptionHandler.process(e);
+                        }
+                    }
+                } catch (Exception e) {
+                    ExceptionHandler.process(e);
+                }
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
+    private void initRedirectList() {
+        try {
+            redirectWhiteList = new HashSet<>();
+            String property = System.getProperty(PROP_ALLOW_PROXY_REDIRECT_EXCLUDE, PROP_ALLOW_PROXY_REDIRECT_EXCLUDE_DEFAULT);
+            if (StringUtils.isEmpty(property)) {
+                return;
+            }
+            String[] split = property.split(";");
+            for (String host : split) {
+                host = StringUtils.strip(host);
+                if (StringUtils.isBlank(host)) {
+                    host = KEY_DEFAULT;
+                }
+                redirectWhiteList.add(host);
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
     }
 
     public static TalendProxySelector getInstance() {
@@ -58,6 +232,18 @@ public class TalendProxySelector extends ProxySelector {
             synchronized (instanceLock) {
                 if (instance == null) {
                     instance = new TalendProxySelector(proxySelector);
+                    try {
+                        uriHostField = URI.class.getDeclaredField("host");
+                        uriHostField.setAccessible(true);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
+                    try {
+                        proxyManagerUpdateSystemPropertiesFunc = ProxyManager.class.getDeclaredMethod("updateSystemProperties");
+                        proxyManagerUpdateSystemPropertiesFunc.setAccessible(true);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
                 }
             }
         }
@@ -70,19 +256,216 @@ public class TalendProxySelector extends ProxySelector {
                     return null;
                 }
             });
-            if (instance.getDefaultProxySelector() == null
+            if (instance.getEclipseDefaultSelector() == null
                     || (proxySelector != null && proxySelector.getClass().getName().endsWith(ECLIPSE_PROXY_SELECTOR))) {
-                instance.setDefaultProxySelector(proxySelector);
+                instance.setEclipseDefaultSelector(proxySelector);
             }
         }
 
         return instance;
     }
 
+    public static void checkProxy() {
+        if (!checkProxy) {
+            return;
+        }
+        try {
+            TalendProxySelector.getInstance();
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
     @Override
     public List<Proxy> select(final URI uri) {
-        final Set<Proxy> resultFromProviders = new HashSet<>();
-        List<IProxySelectorProvider> providers = getProxySelectorProviders();
+        if (printProxyLog) {
+            ExceptionHandler.log("TalendProxySelector.select " + uri);
+        }
+        if (uri == null) {
+            return Collections.EMPTY_LIST;
+        }
+        URI validatedUri = validateUri(uri);
+        Set<Proxy> results = new LinkedHashSet<>();
+
+        try {
+            final Set<Proxy> resultFromProviders = getProxysFromProviders(validatedUri);
+            if (resultFromProviders != null && !resultFromProviders.isEmpty()) {
+                results.addAll(resultFromProviders);
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+        if (printProxyLog) {
+            ExceptionHandler.log("TalendProxySelector.resultFromProviders " + results);
+        }
+
+        ProxySelector defaultProxySelector = getDefaultProxySelector();
+        if (printProxyLog) {
+            ExceptionHandler.log("TalendProxySelector.defaultProxySelector " + defaultProxySelector);
+        }
+        if (defaultProxySelector != null) {
+            /**
+             * don't validate uri here, so that we can know whether it is an issue uri
+             */
+            URI newUri = getNewUri(validatedUri, false);
+            List<Proxy> defaultProxys = null;
+            if (validateUri && StringUtils.isBlank(newUri.getHost())) {
+                /**
+                 * If host is blank, force to use jre proxy selector to avoid the eclipse proxy selector bug
+                 */
+                defaultProxys = getJreProxySelector().select(newUri);
+            } else {
+                defaultProxys = defaultProxySelector.select(newUri);
+            }
+            if (printProxyLog) {
+                ExceptionHandler.log("TalendProxySelector.defaultProxys " + defaultProxys);
+            }
+            try {
+                results.addAll(filterProxys(validatedUri, defaultProxys));
+            } catch (Exception e) {
+                results.addAll(defaultProxys);
+                ExceptionHandler.process(e);
+            }
+        }
+        if (printProxyLog) {
+            String proxys = results.toString();
+            ExceptionHandler.log("Selected proxys for " + uri + ", " + proxys);
+            ExceptionHandler.process(new Exception("Proxy call stacks"), Priority.INFO);
+        }
+        return new LinkedList<Proxy>(results);
+    }
+
+    private URI validateUri(URI uri) {
+        if (!validateUri) {
+            return uri;
+        }
+
+        URI validatedUri = null;
+        try {
+            /**
+             * DON'T use URI.create(), MUST use the conductor which requires authority
+             */
+            validatedUri = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), uri.getQuery(), uri.getFragment());
+
+            /**
+             * Validate the host, if the host is empty, it will cause the eclipse selector to return dirrect
+             */
+            if (StringUtils.isBlank(validatedUri.getHost())) {
+                String authority = validatedUri.getAuthority();
+                if (StringUtils.isNotBlank(authority)) {
+                    // example: https://u:p@www.company.com:8081/path/a?param=b
+                    String host = null;
+                    int userInfoIndex = authority.indexOf('@');
+                    if (0 <= userInfoIndex) {
+                        authority = authority.substring(userInfoIndex + 1);
+                    }
+                    int portIndex = authority.lastIndexOf(':');
+                    if (0 <= portIndex) {
+                        host = authority.substring(0, portIndex);
+                    }
+                    try {
+                        uriHostField.set(validatedUri, host);
+                    } catch (Exception e) {
+                        ExceptionHandler.process(e);
+                    }
+                }
+
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+            if (validatedUri == null) {
+                validatedUri = uri;
+            }
+        }
+        if (printProxyLog) {
+            ExceptionHandler.log("After validate: " + uri + " -> " + validatedUri);
+        }
+        return validatedUri;
+    }
+
+    private List<Proxy> filterProxys(final URI uri, List<Proxy> defaultProxys) {
+        List<Proxy> result = new ArrayList<>();
+        if (defaultProxys != null && !defaultProxys.isEmpty()) {
+            for (Proxy proxy : defaultProxys) {
+                SocketAddress addr = null;
+                Proxy.Type proxyType = null;
+                if (proxy != null) {
+                    proxyType = proxy.type();
+                    addr = proxy.address();
+                }
+
+                boolean redirect = true;
+                if (!allowProxyRedirect) {
+                    String host = uri.getHost();
+                    if (host == null) {
+                        host = "";
+                    }
+                    host = StringUtils.strip(host).toLowerCase();
+                    if (this.redirectWhiteList.contains(host) || this.redirectWhiteList.contains(KEY_DEFAULT)) {
+                        redirect = true;
+                    } else if (Proxy.Type.DIRECT == proxyType
+                            || (addr != null && StringUtils.equals(uri.getHost(), ((InetSocketAddress) addr).getHostString()))) {
+                        redirect = false;
+                    }
+                }
+                if (redirect) {
+                    result.add(proxy);
+                } else {
+                    result.add(Proxy.NO_PROXY);
+                }
+            }
+        }
+        return result;
+    }
+
+    private URI getNewUri(URI uri, boolean validateUri) {
+        URI newUri = uri;
+        if (newUri != null) {
+            try {
+                // get host before new URI, because the host may be set manually due to URI issue
+                String host = newUri.getHost();
+                newUri = new URI(newUri.getScheme(), newUri.getAuthority(), newUri.getPath(), newUri.getQuery(),
+                        newUri.getFragment());
+                Map<String, String> protocolMap = null;
+                if (StringUtils.isNotBlank(host)) {
+                    protocolMap = hostMap.get(host.toLowerCase());
+                }
+                if (protocolMap == null) {
+                    protocolMap = hostMap.get(KEY_DEFAULT);
+                }
+
+                if (protocolMap != null) {
+                    String schema = newUri.getScheme();
+                    if (schema != null) {
+                        String lowercasedProtocol = schema.toLowerCase();
+                        String preferedProtocol = protocolMap.get(lowercasedProtocol);
+                        if (StringUtils.isNotBlank(preferedProtocol)) {
+                            /**
+                             * Note: MUST use the constructor which requires authority, because some uri may be illegal,
+                             * then host info will be stored in authority field instead of host filed
+                             */
+                            newUri = new URI(preferedProtocol, newUri.getAuthority(), newUri.getPath(), newUri.getQuery(),
+                                    newUri.getFragment());
+                        }
+                    }
+                }
+            } catch (URISyntaxException e) {
+                if (printProxyLog) {
+                    ExceptionHandler.process(
+                            new Exception(Messages.getString("TalendProxySelector.exception.proxySelectionError", uri), e),
+                            Priority.WARN);
+                }
+            }
+        }
+        if (validateUri) {
+            newUri = validateUri(newUri);
+        }
+        return newUri;
+    }
+
+    private Set<Proxy> getProxysFromProviders(final URI uri) {
+        final Set<Proxy> resultFromProviders = new LinkedHashSet<>();
+        Collection<IProxySelectorProvider> providers = getCustomProviders(uri);
         if (providers != null) {
             providers.stream().forEach(p -> {
                 if (instance == p) {
@@ -100,49 +483,126 @@ public class TalendProxySelector extends ProxySelector {
                 }
             });
         }
-        List<Proxy> result = new ArrayList<>();
+        return resultFromProviders;
+    }
 
-        if (resultFromProviders != null && !resultFromProviders.isEmpty()) {
-            result.addAll(resultFromProviders);
-        }
-
-        ProxySelector defaultProxySelector = getDefaultProxySelector();
-        if (defaultProxySelector != null) {
-            List<Proxy> defaultProxys = defaultProxySelector.select(uri);
-            if (defaultProxys != null && !defaultProxys.isEmpty()) {
-                result.addAll(defaultProxys);
+    private Collection<IProxySelectorProvider> getCustomProviders(final URI uri) {
+        Collection<IProxySelectorProvider> providers = Collections.EMPTY_LIST;
+        Collection<Object> possibleKeys = getPossibleKeys(uri);
+        for (Object key : possibleKeys) {
+            providers = this.selectorProviders.get(key);
+            if (providers != null) {
+                break;
             }
         }
-        return result;
+        return providers;
+    }
+
+    public PasswordAuthentication getHttpPasswordAuthentication() {
+        String[] schemas = new String[] { "http", "https" };
+        for (String schema : schemas) {
+            String proxyUser = System.getProperty(schema + ".proxyUser");
+            String proxyPassword = System.getProperty(schema + ".proxyPassword");
+
+            if (StringUtils.isNotBlank(proxyUser)) {
+                char[] pwdChars = new char[0];
+                if (proxyPassword != null && !proxyPassword.isEmpty()) {
+                    pwdChars = proxyPassword.toCharArray();
+                }
+                return new PasswordAuthentication(proxyUser, pwdChars);
+            }
+        }
+        return null;
     }
 
     public boolean addProxySelectorProvider(IProxySelectorProvider provider) {
-        List<IProxySelectorProvider> proxySelectorProviders = getProxySelectorProviders();
-        if (!proxySelectorProviders.contains(provider)) {
-            return proxySelectorProviders.add(provider);
+        try {
+            Object key = provider.getKey();
+            Collection<IProxySelectorProvider> collection = this.selectorProviders.get(key);
+            if (collection == null) {
+                synchronized (this.selectorProviders) {
+                    collection = this.selectorProviders.get(key);
+                    if (collection == null) {
+                        collection = Collections.synchronizedList(new LinkedList<>());
+                        this.selectorProviders.put(key, collection);
+                    }
+                }
+            }
+            collection.add(provider);
+            return true;
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
         }
         return false;
     }
 
     public boolean removeProxySelectorProvider(IProxySelectorProvider provider) {
-        return getProxySelectorProviders().remove(provider);
-    }
-
-    private List<IProxySelectorProvider> getProxySelectorProviders() {
-        return selectorProviders;
+        try {
+            Object key = provider.getKey();
+            Collection<IProxySelectorProvider> collection = this.selectorProviders.get(key);
+            if (collection != null) {
+                synchronized (this.selectorProviders) {
+                    collection = this.selectorProviders.get(key);
+                    if (collection != null) {
+                        collection.remove(provider);
+                        if (collection.isEmpty()) {
+                            this.selectorProviders.remove(key);
+                        }
+                    }
+                }
+            }
+            return true;
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+        return false;
     }
 
     public ProxySelector getDefaultProxySelector() {
-        return defaultSelector;
+        switch (eProxySelector) {
+        case jre:
+            return getJreProxySelector();
+        default:
+            return eclipseDefaultSelector;
+        }
     }
 
-    public void setDefaultProxySelector(final ProxySelector selector) {
-        defaultSelector = selector;
+    private ProxySelector getJreProxySelector() {
+        try {
+            /**
+             * for tcompv0, daikon may create an isolated system properties for it, so proxies may be ignored in the new
+             * system properties; here we try to call the method to add proxies into the isolated system properties
+             */
+            if (updateSystemPropertiesForJre && ClassLoaderIsolatedSystemProperties.getInstance()
+                    .isIsolated(Thread.currentThread().getContextClassLoader())) {
+                if (printProxyLog) {
+                    ExceptionHandler.log("Before update jre proxy system properties for the isolated classloader, http.proxyHost="
+                            + System.getProperty("http.proxyHost"));
+                }
+                IProxyService proxyService = CommonsPlugin.getProxyService();
+                proxyManagerUpdateSystemPropertiesFunc.invoke(proxyService);
+                if (printProxyLog) {
+                    ExceptionHandler.log("After updated jre proxy system properties for the isolated classloader, http.proxyHost="
+                            + System.getProperty("http.proxyHost"));
+                }
+            }
+        } catch (Throwable e) {
+            ExceptionHandler.process(e);
+        }
+        return this.jreDefaultSelector;
+    }
+
+    public ProxySelector getEclipseDefaultSelector() {
+        return eclipseDefaultSelector;
+    }
+
+    public void setEclipseDefaultSelector(ProxySelector eclipseDefaultSelector) {
+        this.eclipseDefaultSelector = eclipseDefaultSelector;
     }
 
     @Override
     public void connectFailed(final URI uri, final SocketAddress sa, final IOException ioe) {
-        List<IProxySelectorProvider> providers = getProxySelectorProviders();
+        Collection<IProxySelectorProvider> providers = getCustomProviders(uri);
         if (providers != null) {
             providers.stream().forEach(p -> {
                 if (p.canHandle(uri)) {
@@ -151,23 +611,35 @@ public class TalendProxySelector extends ProxySelector {
             });
         }
 
-        ProxySelector defaultProxySelector = getDefaultProxySelector();
-        if (defaultProxySelector != null) {
-            defaultProxySelector.connectFailed(uri, sa, ioe);
+        if (executeConnectionFailed) {
+            /**
+             * Just try to make the behavior of jre proxy selector same like eclipse proxy selector
+             */
+            ProxySelector defaultProxySelector = getDefaultProxySelector();
+            if (defaultProxySelector != null) {
+                defaultProxySelector.connectFailed(uri, sa, ioe);
+            }
         }
     }
 
-    public static abstract class AbstractProxySelectorProvider implements IProxySelectorProvider {
-
-        @Override
-        public void connectFailed(final URI uri, final SocketAddress sa, final IOException ioe) {
-            // nothing to do
+    public IProxySelectorProvider createDefaultProxySelectorProvider() {
+        if (disableDefaultSelectorProvider) {
+            return null;
         }
+        return new DefaultProxySelectorProvider(Thread.currentThread());
+    }
 
+    public IProxySelectorProvider createDefaultProxySelectorProvider(String host) {
+        if (disableDefaultSelectorProvider) {
+            return null;
+        }
+        return new DefaultProxySelectorProvider(host);
     }
 
     public static interface IProxySelectorProvider {
 
+        Object getKey();
+
         boolean canHandle(final URI uri);
 
         List<Proxy> select(final URI uri);
@@ -176,4 +648,120 @@ public class TalendProxySelector extends ProxySelector {
 
     }
 
+    public static abstract class AbstractProxySelectorProvider implements IProxySelectorProvider {
+
+        private boolean isDebugMode = CommonsPlugin.isDebugMode();
+
+        @Override
+        public Object getKey() {
+            return this;
+        }
+
+        @Override
+        public void connectFailed(final URI uri, final SocketAddress sa, final IOException ioe) {
+            if (isDebugMode) {
+                ExceptionHandler.process(ioe);
+            }
+        }
+
+    }
+
+    public static Collection<Object> getPossibleKeys(URI uri) {
+        Collection<Object> possibleKeys = new ArrayList<>();
+        possibleKeys.add(Thread.currentThread());
+        if (uri != null) {
+            String uriHost = uri.getHost();
+            if (StringUtils.isNotBlank(uriHost)) {
+                possibleKeys.add(uriHost);
+            }
+        }
+        return possibleKeys;
+    }
+
+    private enum EProxySelector {
+        eclipse_default,
+        jre
+    }
+
+    private class DefaultProxySelectorProvider extends TalendProxySelector.AbstractProxySelectorProvider {
+
+        private Thread currentThread = null;
+
+        private String host = null;
+
+        public DefaultProxySelectorProvider(Thread thread) {
+            this.currentThread = thread;
+        }
+
+        public DefaultProxySelectorProvider(String host) {
+            this.host = host;
+            if (StringUtils.isNotBlank(this.host)) {
+                this.host = this.host.toLowerCase();
+            }
+        }
+
+        @Override
+        public Object getKey() {
+            if (this.currentThread != null) {
+                return currentThread;
+            }
+            if (this.host != null) {
+                return this.host;
+            }
+            return super.getKey();
+        }
+
+        @Override
+        public boolean canHandle(URI uri) {
+            if (disableDefaultSelectorProvider) {
+                return false;
+            }
+            if (currentThread != null && Thread.currentThread() == currentThread) {
+                return true;
+            }
+            if (host != null) {
+                if (uri == null) {
+                    return false;
+                }
+                String uriHost = uri.getHost();
+                if (StringUtils.isNotBlank(uriHost)) {
+                    return this.host.equals(uriHost.toLowerCase());
+                }
+                return false;
+            }
+            return false;
+        }
+
+        @Override
+        public List<Proxy> select(URI uri) {
+            List<Proxy> result = new ArrayList<>();
+            try {
+                ProxySelector defaultProxySelector = getDefaultProxySelector();
+                if (defaultProxySelector != null) {
+                    List<Proxy> defaultProxys = defaultProxySelector.select(uri);
+                    if (defaultProxys != null && !defaultProxys.isEmpty()) {
+                        for (Proxy proxy : defaultProxys) {
+                            SocketAddress addr = null;
+                            Proxy.Type proxyType = null;
+                            if (proxy != null) {
+                                proxyType = proxy.type();
+                                addr = proxy.address();
+                            }
+                            if (Proxy.Type.DIRECT == proxyType || (addr != null
+                                    && StringUtils.equals(uri.getHost(), ((InetSocketAddress) addr).getHostString()))) {
+                                result.add(Proxy.NO_PROXY);
+                            } else {
+                                result.add(proxy);
+                            }
+                        }
+                    }
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+            return result;
+        }
+
+    }
+
 }

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/resource/UpdatesHelper.java

@@ -58,6 +58,8 @@ public class UpdatesHelper {
 
     public static final String COMPONENT_SUFFIX = "_java.xml";
 
+    public static final String COMPONENT_TEMPLATES = "templates";
+
     public static final String NEW_COMPONENT_PREFIX = "installer$$";
 
     public static boolean existArtifacts(File base) {
@@ -296,6 +298,9 @@ public class UpdatesHelper {
 
     public static boolean isOldComponent(File f) {
         if (f != null && f.exists() && f.isDirectory()) {
+            if (f.getName().equals(COMPONENT_TEMPLATES)) {
+                return true;
+            }
             File[] listFiles = f.listFiles();
             if (listFiles != null) {
                 for (File subFile : listFiles) {

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/drawing/background/BackgroundRefresher.java

@@ -339,7 +339,7 @@ public class BackgroundRefresher implements IBackgroundRefresher {
             }
             if ((WindowSystem.isGTK() || EnvironmentUtils.isMacOsSytem()) && child instanceof Tree) {
                 returnedPoint.y += ((Tree) child).getHeaderHeight();
-                returnedPoint.y += ((Table) child).getItemHeight();
+                returnedPoint.y += ((Tree) child).getItemHeight();
             }
             child = child.getParent();
             ScrollBar vScrollBar = child.getVerticalBar();

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/linking/TableToTreeLinker.java

@@ -104,7 +104,6 @@ public class TableToTreeLinker<D1, D2> extends BgDrawableComposite implements IB
         this.source = sourceTable;
 
         dataToTableItemCache = new DataToTableItemCache(sourceTable);
-
     }
 
     protected IStyleLink getDefaultStyleLink() {
@@ -232,7 +231,7 @@ public class TableToTreeLinker<D1, D2> extends BgDrawableComposite implements IB
 
             int yStraight = sourceToCommonPoint.y + treeItemHeight / 2 + tableItemBounds.y;
             pointEndStraight.x = sourceToCommonPoint.x + xStartBezierLink;
-            if (Platform.OS_MACOSX.equals(Platform.getOS())) {
+            if (Platform.OS_MACOSX.equals(Platform.getOS()) || Platform.OS_LINUX.equals(Platform.getOS())) {
                 pointStartStraight.x = sourceToCommonPoint.x + tableItem.getParent().getBounds().width;
                 pointEndStraight.x = pointStartStraight.x;
             } else {
@@ -311,10 +310,10 @@ public class TableToTreeLinker<D1, D2> extends BgDrawableComposite implements IB
                 // Added by Marvin Wang on Nov. 28, 2012 for bug TDI-23378. This is not the best way to fix this issue,
                 // but till now I have not found the root cause.
                 if (Platform.OS_LINUX.equals(Platform.getOS())) {
-                    pointEndCentralCurve.y = pointEndCentralCurve.y - tableItem.getBounds().height;
+                    pointEndCentralCurve.y = pointEndCentralCurve.y - tableItem.getBounds().height - treeItemHeight / 2;
                 }
                 if (Platform.OS_MACOSX.equals(Platform.getOS())) {
-                    pointEndCentralCurve.y = pointEndCentralCurve.y + tableItem.getBounds(0).height;
+                    pointEndCentralCurve.y = pointEndCentralCurve.y - tableItem.getBounds().height;
                 }
                 drawableLink.setPoint1(pointEndStraight);
                 drawableLink.setPoint2(pointEndCentralCurve);

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java

@@ -70,6 +70,7 @@ import org.talend.commons.runtime.service.ITaCoKitService;
 import org.talend.commons.ui.gmf.util.DisplayUtils;
 import org.talend.commons.ui.runtime.exception.MessageBoxExceptionHandler;
 import org.talend.commons.utils.data.container.RootContainer;
+import org.talend.commons.utils.network.TalendProxySelector;
 import org.talend.commons.utils.time.TimeMeasure;
 import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.AbstractDQModelService;
@@ -2020,6 +2021,11 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
 
             TimeMeasure.begin("logOnProject"); //$NON-NLS-1$
             try {
+                /**
+                 * init/check proxy selector, in case default proxy selector is not registed yet
+                 */
+                TalendProxySelector.checkProxy();
+
                 System.getProperties().put("ReadOnlyUser", Boolean.FALSE.toString()); //$NON-NLS-1$
 
                 // remove the auto-build to enhance the build speed and application's use
@@ -2405,9 +2411,14 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
     @Override
     @SuppressWarnings("unchecked")
     public void executeRepositoryWorkUnit(RepositoryWorkUnit workUnit) {
+        checkProxySettings();
         this.repositoryFactoryFromProvider.executeRepositoryWorkUnit(workUnit);
     }
 
+    private void checkProxySettings() {
+        TalendProxySelector.checkProxy();
+    }
+
     @Override
     public void unloadResources(Property property) throws PersistenceException {
         repositoryFactoryFromProvider.unloadResources(property);

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/recyclebin/RecycleBinManager.java

@@ -14,6 +14,7 @@ package org.talend.core.repository.recyclebin;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
@@ -24,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.StringUtils;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.emf.common.util.EList;
@@ -106,7 +108,8 @@ public class RecycleBinManager {
         List<IRepositoryViewObject> deletedObjects = new ArrayList<IRepositoryViewObject>();
         final EList<TalendItem> deletedItems = projectRecyclebins.get(project.getTechnicalLabel()).getDeletedItems();
         List<TalendItem> notDeletedItems = new ArrayList<TalendItem>();
-        for (TalendItem deletedItem : deletedItems) {
+        List<TalendItem> dup_deletedItems = new ArrayList<>(deletedItems);
+        for (TalendItem deletedItem : dup_deletedItems) {
             try {
                 final ERepositoryObjectType type = ERepositoryObjectType.getType(deletedItem.getType());
                 // ignore the generated doc in recycle bin
@@ -274,7 +277,40 @@ public class RecycleBinManager {
                 resource = createRecycleBinResource(project);
             }
             resource.getContents().clear();
-            recycleBin.setLastUpdate(new Date());
+            EList<String> deletedFolders = recycleBin.getDeletedFolders();
+            if (deletedFolders != null) {
+                List<String> folders = new LinkedList<>(deletedFolders);
+                Collections.sort(folders);
+                deletedFolders.clear();
+                deletedFolders.addAll(folders);
+            }
+            EList<TalendItem> deletedItems = recycleBin.getDeletedItems();
+            if (deletedItems != null) {
+                List<TalendItem> items = new LinkedList<>(deletedItems);
+                items.sort((l, r) -> {
+                    if (l == null && r == null) {
+                        return 0;
+                    } else if (l == null) {
+                        return -1;
+                    } else if (r == null) {
+                        return 1;
+                    }
+                    int result = StringUtils.compare(l.getType(), r.getType());
+                    if (result != 0) {
+                        return result;
+                    }
+                    result = StringUtils.compare(l.getPath(), r.getPath());
+                    if (result != 0) {
+                        return result;
+                    }
+                    return StringUtils.compare(l.getId(), r.getId());
+                });
+                deletedItems.clear();
+                deletedItems.addAll(items);
+            }
+
+            // set date to null to avoid timezone conflict
+            recycleBin.setLastUpdate(null);
             resource.getContents().add(recycleBin);
             EmfHelper.saveResource(resource);
             lastSavedRecycleBinMap.put(recycleBin, EcoreUtil.copy(recycleBin));

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/ProjectHelper.java

@@ -131,4 +131,22 @@ public class ProjectHelper {
         }
         return null;
     }
+
+    /**
+     * define the different value for the project types the string value in the licence files Matcht the java enum name
+     * this is a copy of org.talend.commons.model.KeyConstants.ProjectType to avoid adding the all library
+     *
+     */
+    public static enum ProjectType {
+        DI,
+        DQ,
+        MDM;
+    }
+
+    public static int getProjectTypeOrdinal(org.talend.core.model.properties.Project project) {
+        if (project != null && project.getType() != null) {
+            return ProjectType.valueOf(project.getType()).ordinal();
+        }
+        return 0;
+    }
 }

main/plugins/org.talend.core.runtime/META-INF/MANIFEST.MF

@@ -120,7 +120,8 @@ Require-Bundle: org.eclipse.jdt.core,
  org.talend.components.api,
  org.apache.servicemix.bundles.avro,
  jackson-core-asl,
- org.talend.libraries.jackson
+ org.talend.libraries.jackson,
+ org.eclipse.m2e.core
 Bundle-Activator: org.talend.core.runtime.CoreRuntimePlugin
 Bundle-ActivationPolicy: lazy
 Bundle-ClassPath: .,

main/plugins/org.talend.core.runtime/mappings/mapping_XSD.xml

@@ -29,6 +29,8 @@
 			<dbType type="xs:time" ignoreLen="true" ignorePre="true"/>
 			<dbType type="xs:decimal" ignoreLen="true" ignorePre="true"/>
 			<dbType type="xs:QName" ignoreLen="true" ignorePre="true"/>
+			<dbType type="xs:list" ignoreLen="true" ignorePre="true"/>
+			<dbType type="xs:union" ignoreLen="true" ignorePre="true"/>
 		</dbTypes>
 
 		<language name="java">
@@ -141,6 +143,12 @@
 				<dbType type="xs:time">
 					<talendType type="id_Date" default="true" />
 				</dbType>
+				<dbType type="xs:list">
+					<talendType type="id_List" default="true" />
+				</dbType>
+				<dbType type="xs:union">
+					<talendType type="id_Object" default="true" />
+				</dbType>
 			</dbToTalendTypes>
 		</language>
 	</dbms>

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/IESBService.java

@@ -12,11 +12,14 @@
 // ============================================================================
 package org.talend.core;
 
+import java.util.List;
+
 import org.eclipse.ui.IEditorPart;
 import org.eclipse.ui.IWorkbenchPart;
 import org.talend.core.model.process.INode;
 import org.talend.core.model.process.IProcess;
 import org.talend.core.model.properties.Item;
+import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.utils.IXSDPopulationUtil;
@@ -47,6 +50,8 @@ public interface IESBService extends IService {
 
     public StringBuffer getAllTheJObNames(IRepositoryNode jobList);
 
+    public List<String> getSerivceRelatedJobIds(Item serviceItem);
+
     public void deleteOldRelation(String jobID);
 
     // public void setSelectedItem(Item, )
@@ -59,6 +64,8 @@ public interface IESBService extends IService {
 
     public void copyDataServiceRelateJob(Item newItem);
 
+    public boolean isRESTService(ProcessItem processItem);
+
     public IXSDPopulationUtil getXSDPopulationUtil();
 
     public boolean isWSDLEditor(IWorkbenchPart part);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ILibraryManagerService.java

@@ -206,8 +206,12 @@ public interface ILibraryManagerService extends IService {
      */
     public boolean isJarNeedToBeDeployed(File jarFile);
 
+    public boolean isSameFile(File f1, File f2);
+
     public void checkModuleStatus(ModuleNeeded module);
 
     public String getJarNameFromMavenuri(String mavenURI);
 
+    public void guessMavenRUIFromIndex(File jarFile, Map<String, String> sourceAndMavenUri);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/EDatabaseTypeName.java

@@ -53,6 +53,8 @@ public enum EDatabaseTypeName {
             "IBM DB2 ZOS", "IBM DB2 ZOS", Boolean.TRUE, "IBM_DB2", "DB2", EDatabaseSchemaOrCatalogMapping.Sid, EDatabaseSchemaOrCatalogMapping.Schema), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
     SYBASEASE(
             "SybaseASE", "Sybase (ASE and IQ)", Boolean.TRUE, "SYBASE", EDatabaseSchemaOrCatalogMapping.Sid, EDatabaseSchemaOrCatalogMapping.None), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+    SYBASEASE_SA(
+            "SybaseASE", "SQL Anywhere", Boolean.TRUE, "SYBASE", EDatabaseSchemaOrCatalogMapping.Sid, EDatabaseSchemaOrCatalogMapping.None), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
 
     // this Sybase IQ not used.
     SYBASEIQ(

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/ConnParameterKeys.java

@@ -302,4 +302,11 @@ public class ConnParameterKeys {
 
     public static final String CONN_PARA_KEY_DATABRICKS_DBFS_DEP_FOLDER="CONN_PARA_KEY_DATABRICKS_DBFS_DEP_FOLDER";
 
+    /**
+     * Override hadoop configuration
+     */
+    public static final String CONN_PARA_KEY_SET_HADOOP_CONF = "CONN_PARA_KEY_SET_HADOOP_CONF";
+
+    public static final String CONN_PARA_KEY_HADOOP_CONF_SPECIFIC_JAR = "CONN_PARA_KEY_HADOOP_CONF_SPECIFIC_JAR";
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/DatabaseConnStrUtil.java

@@ -47,6 +47,8 @@ public class DatabaseConnStrUtil {
     // for match url has :<port> exist
     private static final String PATTERN_PORT = "(:\\d{1,5})";
 
+    private static final String DATABASE_STRING = "DATABASE=";
+
     private static String getStringReplace(final String init, final String before, final String after,
             final boolean supportContext) {
         return getStringReplace(init, before, after, supportContext, false);
@@ -127,6 +129,13 @@ public class DatabaseConnStrUtil {
                 } else {
                     s = getStringReplace(s, EDatabaseConnVar.PORT.getVariable(), port, supportContext);
                 }
+                if (EDatabaseConnTemplate.TERADATA.equals(connStr)) {
+                    if (StringUtils.isNotBlank(sid)) {
+                        s = getStringReplace(s, EDatabaseConnVar.SID.getVariable(), DATABASE_STRING + sid, supportContext);
+                    } else {
+                        s = getStringReplace(s, EDatabaseConnVar.SID.getVariable() + ",", sid, supportContext); //$NON-NLS-1$
+                    }
+                }
                 s = getStringReplace(s, EDatabaseConnVar.SID.getVariable(), sid, supportContext);
                 s = getStringReplace(s, EDatabaseConnVar.SERVICE_NAME.getVariable(), sid, supportContext);
                 s = getStringReplace(s, EDatabaseConnVar.DATASOURCE.getVariable(), datasource, supportContext);
@@ -565,7 +574,7 @@ public class DatabaseConnStrUtil {
         List<ERepositoryObjectType> extraTypes = new ArrayList<>();
         IGenericDBService dbService = null;
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericDBService.class)) {
-            dbService = (IGenericDBService) GlobalServiceRegister.getDefault().getService(IGenericDBService.class);
+            dbService = GlobalServiceRegister.getDefault().getService(IGenericDBService.class);
         }
         if (dbService != null) {
             extraTypes.addAll(dbService.getExtraTypes());

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/version/EDatabaseVersion4Drivers.java

@@ -166,9 +166,9 @@ public enum EDatabaseVersion4Drivers {
     REDSHIFT(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT, "redshift", "REDSHIFT", //$NON-NLS-1$ //$NON-NLS-2$
             "redshift-jdbc42-no-awssdk-1.2.20.1043.jar")), //$NON-NLS-1$
     REDSHIFT_SSO(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT_SSO, "redshift sso", "REDSHIFT_SSO", //$NON-NLS-1$ //$NON-NLS-2$
-            new String[] { "redshift-jdbc42-no-awssdk-1.2.20.1043.jar", "aws-java-sdk-1.11.406.jar", "jackson-core-2.9.5.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-                    "jackson-databind-2.9.5.jar", "jackson-annotations-2.9.0.jar", "httpcore-4.4.9.jar", "httpclient-4.5.5.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$//$NON-NLS-4$
-                    "joda-time-2.8.1.jar", "commons-logging-1.1.3.jar" })), //$NON-NLS-1$ //$NON-NLS-2$
+            new String[] { "redshift-jdbc42-no-awssdk-1.2.20.1043.jar", "aws-java-sdk-1.11.406.jar", "jackson-core-2.9.9.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+                    "jackson-databind-2.9.9.jar", "jackson-annotations-2.9.0.jar", "httpcore-4.4.9.jar", "httpclient-4.5.5.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$//$NON-NLS-4$
+                    "joda-time-2.8.1.jar", "commons-logging-1.1.3.jar", "commons-codec-1.6.jar" })), //$NON-NLS-1$ //$NON-NLS-2$
 
     AMAZON_AURORA(new DbVersion4Drivers(EDatabaseTypeName.AMAZON_AURORA, "mysql-connector-java-5.1.30-bin.jar")); //$NON-NLS-1$
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/HadoopClassLoaderFactory2.java

@@ -12,9 +12,13 @@
 // ============================================================================
 package org.talend.core.hadoop;
 
+import java.io.File;
 import java.net.MalformedURLException;
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Consumer;
 
+import org.apache.commons.lang.StringUtils;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.classloader.ClassLoaderFactory;
 import org.talend.core.classloader.DynamicClassLoader;
@@ -67,9 +71,18 @@ public class HadoopClassLoaderFactory2 {
         return getClassLoader(relatedClusterId, EHadoopCategory.HBASE, distribution, version, useKrb);
     }
 
-    public static ClassLoader getHadoopCustomClassLoader(String uid, Object customJars) {
-        return HadoopClassLoaderFactory2.builder().withTypePrefix(EHadoopCategory.CUSTOM.getName()).withUid(uid)
-                .build(customJars, true);
+    public static ClassLoader getHadoopCustomClassLoader(String uid, String relatedClusterId, EHadoopCategory category,
+            Object customJars, boolean useKrb) {
+        ClassLoader classLoader = HadoopClassLoaderFactory2.builder().withTypePrefix(EHadoopCategory.CUSTOM.getName())
+                .withUid(uid).build(customJars, true);
+        if (StringUtils.isNotBlank(relatedClusterId) && classLoader instanceof DynamicClassLoader) {
+            try {
+                classLoader = addExtraJars(relatedClusterId, category, (DynamicClassLoader) classLoader, useKrb);
+            } catch (MalformedURLException e) {
+                ExceptionHandler.process(e);
+            }
+        }
+        return classLoader;
     }
 
     public static ClassLoader getHCatalogClassLoader(String relatedClusterId, String distribution, String version, boolean useKrb) {
@@ -198,16 +211,35 @@ public class HadoopClassLoaderFactory2 {
             String[] addedJars = null;
             String[] excludedJars = null;
             String[] securityJars = getSecurityJars(category);
-            String customConfsJarName = hadoopClusterService.getCustomConfsJarName(relatedClusterId);
-            if (customConfsJarName != null) {
-                addedJars = new String[] { customConfsJarName };
+
+            Optional<HadoopConfJarBean> customConfsJar = hadoopClusterService.getCustomConfsJar(relatedClusterId);
+            HadoopConfJarBean confJarBean = customConfsJar.orElse(null);
+            String customConfsJarName = customConfsJar.map(b -> b.getCustomConfJarName()).orElse(null);
+            Consumer<DynamicClassLoader> afterLoaded = null;
+            if (confJarBean != null) {
+                if (confJarBean.isOverrideCustomConf()) {
+                    String overrideCustomConfPath = confJarBean.getOriginalOverrideCustomConfPath();
+                    if (StringUtils.isBlank(overrideCustomConfPath) || !new File(overrideCustomConfPath).exists()) {
+                        ExceptionHandler.process(
+                                new Exception("Set Hadoop configuration JAR path is invalid: " + overrideCustomConfPath));
+                    } else {
+                        afterLoaded = (t) -> t.addLibrary(overrideCustomConfPath);
+                    }
+                } else {
+                    if (StringUtils.isNotBlank(customConfsJarName)) {
+                        addedJars = new String[] { customConfsJarName };
+                    }
+                }
                 excludedJars = securityJars;
             } else if (useKrb) {
                 addedJars = securityJars;
                 excludedJars = new String[] { customConfsJarName };
             }
-            if (addedJars != null || excludedJars != null) {
+            if (addedJars != null || excludedJars != null || afterLoaded != null) {
                 cll = DynamicClassLoader.createNewOneBaseLoader(loader, addedJars, excludedJars);
+                if (afterLoaded != null) {
+                    afterLoaded.accept(cll);
+                }
             }
         }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/HadoopConfJarBean.java

@@ -0,0 +1,92 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.hadoop;
+
+/**
+ * DOC cmeng  class global comment. Detailled comment
+ */
+public class HadoopConfJarBean {
+
+    private boolean overrideCustomConf;
+
+    private boolean isContextMode;
+
+    /**
+     * value saved in connection, it may be a context
+     */
+    private String overrideCustomConfPath;
+
+    /**
+     * original value of the path, if connection is in context mode, then this value if the default value of the context
+     */
+    private String originalOverrideCustomConfPath;
+
+    private String customConfJarName;
+
+    public HadoopConfJarBean(boolean isContextMode, boolean overrideCustomConf, String overrideCustomConfPath,
+            String originalOverrideCustomConfPath, String customConfJarName) {
+        this.isContextMode = isContextMode;
+        this.overrideCustomConf = overrideCustomConf;
+        this.overrideCustomConfPath = overrideCustomConfPath;
+        this.originalOverrideCustomConfPath = originalOverrideCustomConfPath;
+        this.customConfJarName = customConfJarName;
+    }
+
+    public boolean isContextMode() {
+        return isContextMode;
+    }
+
+    public void setContextMode(boolean isContextMode) {
+        this.isContextMode = isContextMode;
+    }
+
+    public boolean isOverrideCustomConf() {
+        return overrideCustomConf;
+    }
+
+    public void setOverrideCustomConf(boolean overrideCustomConf) {
+        this.overrideCustomConf = overrideCustomConf;
+    }
+
+    public String getOriginalOverrideCustomConfPath() {
+        return originalOverrideCustomConfPath;
+    }
+
+    public void setOriginalOverrideCustomConfPath(String originalOverrideCustomConfPath) {
+        this.originalOverrideCustomConfPath = originalOverrideCustomConfPath;
+    }
+
+    public String getOverrideCustomConfPath() {
+        return overrideCustomConfPath;
+    }
+
+    public void setOverrideCustomConfPath(String overrideCustomConfPath) {
+        this.overrideCustomConfPath = overrideCustomConfPath;
+    }
+
+    public String getCustomConfJarName() {
+        return customConfJarName;
+    }
+
+    public void setCustomConfJarName(String customConfJarName) {
+        this.customConfJarName = customConfJarName;
+    }
+
+    @Override
+    public String toString() {
+        return "HadoopConfJarBean [overrideCustomConf=" + overrideCustomConf + ", isContextMode=" + isContextMode
+                + ", overrideCustomConfPath=" + overrideCustomConfPath + ", originalOverrideCustomConfPath="
+                + originalOverrideCustomConfPath + ", customConfJarName=" + customConfJarName + "]";
+    }
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/IHadoopClusterService.java

@@ -14,6 +14,7 @@ package org.talend.core.hadoop;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import org.eclipse.core.runtime.IPath;
 import org.talend.commons.exception.BusinessException;
@@ -232,11 +233,12 @@ public interface IHadoopClusterService extends IService {
      */
     public Item getHadoopClusterItemById(String id);
 
-    public String getCustomConfsJarName(String id);
+    public Optional<HadoopConfJarBean> getCustomConfsJar(String id);
 
-    public String getCustomConfsJarName(String id, boolean createJarIfNotExist, boolean addExtraIds);
+    public Optional<HadoopConfJarBean> getCustomConfsJar(String id, boolean createJarIfNotExist, boolean addExtraIds);
 
-    public String getCustomConfsJarName(ConnectionItem connectionItem, boolean createJarIfNotExist, boolean addExtraIds);
+    public Optional<HadoopConfJarBean> getCustomConfsJar(ConnectionItem connectionItem, boolean createJarIfNotExist,
+            boolean addExtraIds);
 
     public void useCustomConfsJarIfNeeded(List<ModuleNeeded> modulesNeeded, String clusterId);
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/components/IComponent.java

@@ -60,6 +60,15 @@ public interface IComponent {
 
     public String getOriginalName();
 
+    /**
+     * Only for component display (palette,search)
+     * 
+     * @return
+     */
+    default public String getDisplayName() {
+        return getName();
+    }
+
     public String getLongName();
 
     public String getOriginalFamilyName();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/components/IMultipleComponentConnection.java

@@ -23,6 +23,8 @@ public interface IMultipleComponentConnection {
 
     public String getConnectionType();
 
+    public String getConnectorName();
+
     public void setConnectionType(String connectionType);
 
     public String getNameTarget();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/components/IMultipleComponentItem.java

@@ -14,6 +14,8 @@ package org.talend.core.model.components;
 
 import java.util.List;
 
+import org.talend.core.model.process.INode;
+
 /**
  * DOC nrousseau class global comment. Detailled comment <br/>
  *
@@ -33,4 +35,7 @@ public interface IMultipleComponentItem {
     public String getComponent();
 
     public void setComponent(String component);
+
+    public void updateNode(INode newNode, INode oldNode);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/components/IMultipleComponentManager.java

@@ -50,4 +50,6 @@ public interface IMultipleComponentManager {
 
     public boolean isLookupMode();
 
+    public String getParamSeperator();
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/IMetadataTable.java

@@ -68,6 +68,8 @@ public interface IMetadataTable {
     public boolean sameMetadataAs(IMetadataTable meta);
 
     public boolean sameMetadataAs(IMetadataTable other, int options);
+    
+    public boolean sameMetadataAs(IMetadataTable other, int options, boolean order);
 
     public void sortCustomColumns();
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataTable.java

@@ -243,6 +243,11 @@ public class MetadataTable implements IMetadataTable, Cloneable {
      */
     @Override
     public boolean sameMetadataAs(IMetadataTable input, int options) {
+    	return sameMetadataAs(input, options, false);
+    }
+    
+    @Override
+    public boolean sameMetadataAs(IMetadataTable input, int options, boolean order) {
         if (this == input) {
             return true;
         }
@@ -263,7 +268,7 @@ public class MetadataTable implements IMetadataTable, Cloneable {
                 // test if standard columns (no custom, or same input / output)
                 for (int i = 0; i < inputColumnListWithUnselected.size(); i++) {
                     IMetadataColumn otherColumn = inputColumnListWithUnselected.get(i);
-                    if (isRepository) {
+                    if (isRepository && !order) {
                         boolean exist = false;
                         for (int j = 0; j < thisColumnListWithUnselected.size(); j++) {
                             IMetadataColumn myColumn = thisColumnListWithUnselected.get(j);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataTalendType.java

@@ -76,6 +76,8 @@ public final class MetadataTalendType {
 
     public static final String PROJECT_MAPPING_FOLDER = ".settings/mappings"; //$NON-NLS-1$
 
+    public static final String UPDATED_MAPPING_FILES = "updated.mapping.files"; //$NON-NLS-1$
+
     private static ECodeLanguage codeLanguage;
 
     private static final String[] PERL_TYPES = new String[] { PerlTypesManager.BOOLEAN, PerlTypesManager.DATE,
@@ -476,7 +478,7 @@ public final class MetadataTalendType {
             if (!mappingFolder.exists() || mappingFolder.listFiles().length < 1) {
                 ICoreService service = null;
                 if (GlobalServiceRegister.getDefault().isServiceRegistered(ICoreService.class)) {
-                    service = (ICoreService) GlobalServiceRegister.getDefault().getService(ICoreService.class);
+                    service = GlobalServiceRegister.getDefault().getService(ICoreService.class);
                     service.syncMappingsFileFromSystemToProject();
                 }
             }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolHelper.java

@@ -626,6 +626,7 @@ public final class MetadataToolHelper {
         target.getListColumns().addAll(columnsTAdd);
         target.sortCustomColumns();
         target.setLabel(source.getLabel());
+        target.setDbms(source.getDbms());
         setTargetOriginalColumns(source, target);
         // List<String> originalColumnsList = null;
         // if (source.getOriginalColumns() != null) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/repository/ERepositoryObjectType.java

@@ -427,6 +427,11 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
      */
     public final static ERepositoryObjectType JOBLET = ERepositoryObjectType.valueOf("JOBLET"); //$NON-NLS-1$
 
+    /**
+     * <font color="red">This value may be <b>null</b> in some products, <b>should add NPE check</b></font>
+     */
+    public final static ERepositoryObjectType SERVICES = ERepositoryObjectType.valueOf("SERVICES"); //$NON-NLS-1$
+
     public final static ERepositoryObjectType JOBLET_DESIGNS = ERepositoryObjectType.valueOf("JOBLET_DESIGNS"); //$NON-NLS-1$
 
     public final static ERepositoryObjectType SPARK_JOBLET = ERepositoryObjectType.valueOf("SPARK_JOBLET"); //$NON-NLS-1$
@@ -439,6 +444,8 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public final static ERepositoryObjectType ROUTINES = ERepositoryObjectType.valueOf("ROUTINES"); //$NON-NLS-1$
 
+    public final static ERepositoryObjectType BEANS = ERepositoryObjectType.valueOf("BEANS"); //$NON-NLS-1$
+
     public final static ERepositoryObjectType METADATA_HEADER_FOOTER = ERepositoryObjectType.valueOf("METADATA_HEADER_FOOTER"); //$NON-NLS-1$
 
     public final static ERepositoryObjectType JOB_SCRIPT = ERepositoryObjectType.valueOf("JOB_SCRIPT"); //$NON-NLS-1$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/routines/RoutineLibraryMananger.java

@@ -13,6 +13,7 @@
 package org.talend.core.model.routines;
 
 import java.io.File;
+import java.io.IOException;
 import java.net.URI;
 import java.net.URL;
 import java.util.ArrayList;
@@ -31,9 +32,12 @@ import org.osgi.framework.Bundle;
 import org.talend.commons.utils.resource.FileExtensions;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.ILibraryManagerService;
-import org.talend.core.ISVNProviderServiceInCoreRuntime;
-import org.talend.core.PluginChecker;
 import org.talend.core.model.general.LibraryInfo;
+import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.nexus.TalendLibsServerManager;
+import org.talend.core.runtime.maven.MavenArtifact;
+import org.talend.core.runtime.maven.MavenConstants;
+import org.talend.core.runtime.maven.MavenUrlHelper;
 
 /**
  * wchen class global comment. Detailled comment
@@ -96,9 +100,12 @@ public class RoutineLibraryMananger {
                                 try {
                                     URL fileUrl = FileLocator.toFileURL(entry);
                                     if(fileUrl != null){
-                                    	if (!"file".equals(fileUrl.getProtocol())) throw new IllegalArgumentException();
-                                    	URI fileUri = new File(fileUrl.getFile()).toURI();
-                                        libManagerService.deploy(fileUri);
+                                        if (!"file".equals(fileUrl.getProtocol())) throw new IllegalArgumentException();
+                                        File file = new File(fileUrl.getFile());
+                                        if (needDeploy(fileUrl)) {
+                                            URI fileUri = file.toURI();
+                                            libManagerService.deploy(fileUri);
+                                        }
                                     }
                                 } catch (Exception e) {
                                     log.warn("Cannot deploy: " + bundleName + path);
@@ -114,6 +121,56 @@ public class RoutineLibraryMananger {
         }
     }
 
+    private boolean needDeploy(URL fileUrl) throws IOException, Exception {
+        File file = new File(fileUrl.getFile());
+        ILibraryManagerService libManagerService = null;
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibraryManagerService.class)) {
+            libManagerService = (ILibraryManagerService) GlobalServiceRegister.getDefault()
+                    .getService(ILibraryManagerService.class);
+        }
+        if (libManagerService != null) {
+            Map<String, String> sourceAndMavenUri = new HashMap<>();
+            libManagerService.guessMavenRUIFromIndex(file, sourceAndMavenUri);
+            String mavUri = null;
+            boolean isSnapshot = false;
+            for (String key : sourceAndMavenUri.keySet()) {
+                if (sourceAndMavenUri.get(key).equals(file.getPath())) {
+                    mavUri = key;
+                    break;
+                }
+            }
+            if (mavUri == null) {
+                return true;
+            }
+            final MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(mavUri);
+            if (parseMvnUrl != null) {
+                if (parseMvnUrl.getVersion() != null && parseMvnUrl.getVersion().endsWith(MavenConstants.SNAPSHOT)) {
+                    isSnapshot = true;
+                }
+            }
+            TalendLibsServerManager manager = TalendLibsServerManager.getInstance();
+            ArtifactRepositoryBean customNexusServer = manager.getCustomNexusServer();
+            File jarFile = null;
+            try {
+                jarFile = libManagerService.resolveJar(customNexusServer, mavUri);
+            } catch (Exception ex) {
+                // Ignore here
+            }
+            boolean exist = (jarFile != null && jarFile.exists());
+            if (exist) {
+                if (isSnapshot) {
+                    boolean isSame = libManagerService.isSameFile(jarFile, file);
+                    if (!isSame) {
+                        return true;
+                    }
+                }
+            } else {
+                return true;
+            }
+        }
+        return false;
+    }
+
     public Map<String, List<LibraryInfo>> getRoutineAndJars() {
         if (routineAndJars == null) {
             routineAndJars = new HashMap<String, List<LibraryInfo>>();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/update/RepositoryUpdateManager.java

@@ -1785,7 +1785,7 @@ public abstract class RepositoryUpdateManager {
                     IMetadataManagmentService service = (IMetadataManagmentService) GlobalServiceRegister.getDefault()
                             .getService(IMetadataManagmentService.class);
                     IMetadataTable newMetadataTable = service.convertMetadataTable(newTable);
-                    update = !oldMetadataTable.sameMetadataAs(newMetadataTable, IMetadataColumn.OPTIONS_NONE);
+                    update = !oldMetadataTable.sameMetadataAs(newMetadataTable, IMetadataColumn.OPTIONS_NONE, true);
                     isAddColumn = isAddColumn(newMetadataTable, oldMetadataTable);
                 }
             }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/ParameterValueUtil.java

@@ -77,21 +77,24 @@ public final class ParameterValueUtil {
                 }
             }
         } else if (param.getValue() instanceof List) { // for TABLE
-            List<Map<String, Object>> tableValues = (List<Map<String, Object>>) param.getValue();
-            for (Map<String, Object> line : tableValues) {
-                for (String key : line.keySet()) {
-                    Object cellValue = line.get(key);
-                    if (cellValue instanceof String) { // cell is text so
-                        // rename data if
-                        // needed
-                        String value = (String) cellValue;
-                        if (value.contains(oldName)) {
-                            // line.put(key, value.replaceAll(oldName,
-                            // newName));
-                            // String newValue = renameValues(value, oldName, newName, flag);
-                            String newValue = splitQueryData(oldName, newName, value);
-                            if (!value.equals(newValue)) {
-                                line.put(key, newValue);
+            List tableValues = (List) param.getValue();
+            for (Object current : tableValues) {
+                if (current != null && current instanceof Map) {
+                    Map<String, Object> line = (Map<String, Object>) current;
+                    for (String key : line.keySet()) {
+                        Object cellValue = line.get(key);
+                        if (cellValue instanceof String) { // cell is text so
+                            // rename data if
+                            // needed
+                            String value = (String) cellValue;
+                            if (value.contains(oldName)) {
+                                // line.put(key, value.replaceAll(oldName,
+                                // newName));
+                                // String newValue = renameValues(value, oldName, newName, flag);
+                                String newValue = splitQueryData(oldName, newName, value);
+                                if (!value.equals(newValue)) {
+                                    line.put(key, newValue);
+                                }
                             }
                         }
                     }
@@ -721,13 +724,22 @@ public final class ParameterValueUtil {
                 return true;
             }
         } else if (param.getValue() instanceof List) { // for TABLE
-            List<Map<String, Object>> tableValues = (List<Map<String, Object>>) param.getValue();
-            for (Map<String, Object> line : tableValues) {
-                for (String key : line.keySet()) {
-                    Object cellValue = line.get(key);
-                    if (cellValue instanceof String) { // cell is text so
-                        // test data
-                        if (ParameterValueUtil.valueContains((String) cellValue, name)) {
+            List tableValues = (List) param.getValue();
+            for (Object current : tableValues) {
+                if (current != null) {
+                    if (current instanceof Map) {
+                        Map<String, Object> line = (Map<String, Object>) current;
+                        for (String key : line.keySet()) {
+                            Object cellValue = line.get(key);
+                            if (cellValue instanceof String) { // cell is text so
+                                // test data
+                                if (ParameterValueUtil.valueContains((String) cellValue, name)) {
+                                    return true;
+                                }
+                            }
+                        }
+                    } else if (current instanceof String) {
+                        if (ParameterValueUtil.valueContains((String) current, name)) {
                             return true;
                         }
                     }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/HttpClientTransport.java

@@ -12,14 +12,12 @@
 // ============================================================================
 package org.talend.core.nexus;
 
-import java.net.Authenticator;
 import java.net.InetSocketAddress;
 import java.net.PasswordAuthentication;
 import java.net.Proxy;
 import java.net.Proxy.Type;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
@@ -37,7 +35,6 @@ import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.OperationCanceledException;
 import org.talend.commons.exception.BusinessException;
-import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.utils.network.TalendProxySelector;
 import org.talend.commons.utils.network.TalendProxySelector.IProxySelectorProvider;
 import org.talend.core.runtime.CoreRuntimePlugin;
@@ -167,7 +164,7 @@ public abstract class HttpClientTransport {
         }
     }
 
-    private IProxySelectorProvider addProxy(final DefaultHttpClient httpClient, URI requestURI) {
+    public static IProxySelectorProvider addProxy(final DefaultHttpClient httpClient, URI requestURI) {
         IProxySelectorProvider proxySelectorProvider = null;
         try {
             if (Boolean.valueOf(System.getProperty(PROP_PROXY_HTTP_CLIENT_USE_DEFAULT_SETTINGS, Boolean.FALSE.toString()))) {
@@ -185,10 +182,10 @@ public abstract class HttpClientTransport {
                 }
                 final Proxy finalProxy = usedProxy;
                 InetSocketAddress address = (InetSocketAddress) finalProxy.address();
-                String proxyServer = address.getHostName();
+                String proxyServer = address.getHostString();
                 int proxyPort = address.getPort();
-                PasswordAuthentication proxyAuthentication = Authenticator.requestPasswordAuthentication(proxyServer,
-                        address.getAddress(), proxyPort, "Http Proxy", "Http proxy authentication", null);
+                TalendProxySelector proxySelector = TalendProxySelector.getInstance();
+                PasswordAuthentication proxyAuthentication = proxySelector.getHttpPasswordAuthentication();
                 if (proxyAuthentication != null) {
                     String proxyUser = proxyAuthentication.getUserName();
                     if(StringUtils.isNotBlank(proxyUser)){
@@ -203,7 +200,7 @@ public abstract class HttpClientTransport {
                 }
                 HttpHost proxyHost = new HttpHost(proxyServer, proxyPort);
                 httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost);
-                proxySelectorProvider = createProxySelectorProvider();
+                proxySelectorProvider = proxySelector.createDefaultProxySelectorProvider();
             }
             return proxySelectorProvider;
         } finally {
@@ -213,61 +210,12 @@ public abstract class HttpClientTransport {
         }
     }
 
-    private void removeProxy(IProxySelectorProvider proxySelectorProvider) {
+    public static void removeProxy(IProxySelectorProvider proxySelectorProvider) {
         if (proxySelectorProvider != null) {
             TalendProxySelector.getInstance().removeProxySelectorProvider(proxySelectorProvider);
         }
     }
 
-    private IProxySelectorProvider createProxySelectorProvider() {
-        IProxySelectorProvider proxySelectorProvider = new TalendProxySelector.AbstractProxySelectorProvider() {
-
-            private Thread currentThread = Thread.currentThread();
-
-            @Override
-            public List<Proxy> select(URI uri) {
-                // return Collections.EMPTY_LIST;
-
-                List<Proxy> newProxys = new ArrayList<>();
-                if (uri == null) {
-                    return newProxys;
-                }
-                String schema = uri.getScheme();
-                if (schema != null && schema.toLowerCase().startsWith("socket")) { //$NON-NLS-1$
-                    try {
-                        URI newUri = new URI("https", uri.getUserInfo(), uri.getHost(), uri.getPort(), uri.getPath(),
-                                uri.getQuery(), uri.getFragment());
-                        List<Proxy> proxys = TalendProxySelector.getInstance().getDefaultProxySelector().select(newUri);
-                        if (proxys != null && !proxys.isEmpty()) {
-                            newProxys.addAll(proxys);
-                        } else {
-                            newUri = new URI("http", uri.getUserInfo(), uri.getHost(), uri.getPort(), uri.getPath(),
-                                    uri.getQuery(), uri.getFragment());
-                            proxys = TalendProxySelector.getInstance().getDefaultProxySelector().select(newUri);
-                            if (proxys != null && !proxys.isEmpty()) {
-                                newProxys.addAll(proxys);
-                            }
-                        }
-                    } catch (URISyntaxException e) {
-                        ExceptionHandler.process(e);
-                    }
-                }
-                return newProxys;
-
-            }
-
-            @Override
-            public boolean canHandle(URI uri) {
-                if (Thread.currentThread() == currentThread) {
-                    return true;
-                }
-                return false;
-            }
-
-        };
-        return proxySelectorProvider;
-    }
-
     public void processResponseCode(HttpResponse response) throws BusinessException {
         StatusLine statusLine = response.getStatusLine();
         int responseCode = statusLine.getStatusCode();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/IRepositoryArtifactHandler.java

@@ -61,5 +61,7 @@ public interface IRepositoryArtifactHandler {
     public IRepositoryArtifactHandler clone();
 
     public String getRepositoryURL(boolean isRelease);
+    
+    public String resolveRemoteSha1(MavenArtifact artifact, boolean fromRelease) throws Exception;
 
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/TalendMavenResolver.java

@@ -18,12 +18,16 @@ import java.util.Collection;
 import java.util.Dictionary;
 import java.util.Hashtable;
 
+import org.eclipse.m2e.core.MavenPlugin;
 import org.ops4j.pax.url.mvn.MavenResolver;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.ServiceReference;
 import org.osgi.service.cm.ManagedService;
 import org.osgi.util.tracker.ServiceTracker;
 import org.osgi.util.tracker.ServiceTrackerCustomizer;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.TalendProxySelector;
+import org.talend.commons.utils.network.TalendProxySelector.IProxySelectorProvider;
 import org.talend.core.runtime.CoreRuntimePlugin;
 
 /**
@@ -44,6 +48,8 @@ public class TalendMavenResolver {
 
     private static MavenResolver mavenResolver = null;
 
+    private static final String MVN_USER_SETTING_KEY = "org.ops4j.pax.url.mvn.settings";
+
     /**
      *
      * DOC wchen TalendMavenResolver constructor comment.
@@ -84,6 +90,15 @@ public class TalendMavenResolver {
         if (props == null) {
             props = new Hashtable<String, String>();
         }
+
+        // https://jira.talendforge.org/browse/TUP-26752
+        String configFile = props.get(MVN_USER_SETTING_KEY);
+        if (configFile == null || configFile.trim().isEmpty()) {
+            // set existing user settings file
+            String studioUserSettingsFile = MavenPlugin.getMavenConfiguration().getUserSettingsFile();
+            props.put(MVN_USER_SETTING_KEY, studioUserSettingsFile);
+        }
+
         final BundleContext context = CoreRuntimePlugin.getInstance().getBundle().getBundleContext();
         Collection<ServiceReference<ManagedService>> managedServiceRefs = context.getServiceReferences(ManagedService.class,
                 "(service.pid=org.ops4j.pax.url.mvn)");
@@ -102,7 +117,24 @@ public class TalendMavenResolver {
     }
 
     public static File resolve(String mvnUri) throws IOException, RuntimeException {
-        return getMavenResolver().resolve(mvnUri);
+        TalendProxySelector selectorInstance = null;
+        IProxySelectorProvider proxySelector = null;
+        try {
+            try {
+                selectorInstance = TalendProxySelector.getInstance();
+                proxySelector = selectorInstance.createDefaultProxySelectorProvider();
+                if (proxySelector != null) {
+                    selectorInstance.addProxySelectorProvider(proxySelector);
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+            return getMavenResolver().resolve(mvnUri);
+        } finally {
+            if (proxySelector != null && selectorInstance != null) {
+                selectorInstance.removeProxySelectorProvider(proxySelector);
+            }
+        }
     }
 
     public static void upload(String groupId, String artifactId, String classifier, String extension, String version,

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenArtifact.java

@@ -24,7 +24,7 @@ public class MavenArtifact implements Cloneable {
     private static final char ARTIFACT_SEPARATOR = '-';
 
     private String repositoryUrl, groupId, artifactId, version, type, classifier, description, url, license, licenseUrl,
-            distribution, username, password, lastUpdated;
+            distribution, username, password, lastUpdated, sha1, md5;
 
     public String getLastUpdated() {
         return this.lastUpdated;
@@ -138,6 +138,22 @@ public class MavenArtifact implements Cloneable {
         this.password = password;
     }
 
+    public String getSha1() {
+        return sha1;
+    }
+
+    public void setSha1(String sha1) {
+        this.sha1 = sha1;
+    }
+
+    public String getMd5() {
+        return md5;
+    }
+
+    public void setMd5(String md5) {
+        this.md5 = md5;
+    }
+
     /**
      *
      * DOC ggu Comment method "getFileName".

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenConstants.java

@@ -42,6 +42,8 @@ public interface MavenConstants {
 
     static final String POM_FILTER = "POM_FILTER";
 
+    static final String USE_PROFILE_MODULE = "USE_PROFILE_MODULE";
+
     /*
      * for lib
      */

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/process/LastGenerationInfo.java

@@ -16,6 +16,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.process.JobInfo;
@@ -34,8 +35,12 @@ public class LastGenerationInfo {
 
     private HashMap<String, Set<ModuleNeeded>> modulesNeededWithSubjobPerJob;
 
+    private HashMap<String, Set<ModuleNeeded>> highPriorityModuleNeededPerJob;
+
     private HashMap<String, Set<ModuleNeeded>> highPriorityModuleNeeded;
 
+    private HashMap<String, Set<ModuleNeeded>> testcaseModuleNeeded;
+
     private HashMap<String, Set<String>> routinesNeededWithSubjobPerJob;
 
     private HashMap<String, Set<String>> pigudfNeededWithSubjobPerJob;
@@ -56,7 +61,9 @@ public class LastGenerationInfo {
         modulesNeededPerJob = new HashMap<String, Set<ModuleNeeded>>();
         contextPerJob = new HashMap<String, Set<String>>();
         modulesNeededWithSubjobPerJob = new HashMap<String, Set<ModuleNeeded>>();
+        highPriorityModuleNeededPerJob = new HashMap<>();
         highPriorityModuleNeeded = new HashMap<>();
+        testcaseModuleNeeded = new HashMap<>();
         lastGeneratedjobs = new HashSet<JobInfo>();
         routinesNeededPerJob = new HashMap<String, Set<String>>();
         pigudfNeededPerJob = new HashMap<String, Set<String>>();
@@ -259,6 +266,22 @@ public class LastGenerationInfo {
         return routinesNeededPerJob.get(key);
     }
 
+    public Set<ModuleNeeded> getHighPriorityModuleNeededPerJob(String jobId, String jobVersion) {
+        String key = getProcessKey(jobId, jobVersion);
+        if (!highPriorityModuleNeededPerJob.containsKey(key)) {
+            highPriorityModuleNeededPerJob.put(key, new LinkedHashSet<>());
+        }
+        return highPriorityModuleNeededPerJob.get(key);
+    }
+
+    public void setHighPriorityModuleNeededPerJob(String jobId, String jobVersion, Set<ModuleNeeded> moduleNeeded) {
+        String key = getProcessKey(jobId, jobVersion);
+        if (!highPriorityModuleNeededPerJob.containsKey(key)) {
+            highPriorityModuleNeededPerJob.put(key, new LinkedHashSet<>());
+        }
+        highPriorityModuleNeededPerJob.get(key).addAll(moduleNeeded);
+    }
+
     public Set<ModuleNeeded> getHighPriorityModuleNeeded(String jobId, String jobVersion) {
         String key = getProcessKey(jobId, jobVersion);
         if (!highPriorityModuleNeeded.containsKey(key)) {
@@ -275,7 +298,20 @@ public class LastGenerationInfo {
         highPriorityModuleNeeded.get(key).addAll(moduleNeeded);
     }
 
+    public Set<ModuleNeeded> getTestcaseModuleNeeded(String jobId, String jobVersion) {
+        String key = getProcessKey(jobId, jobVersion);
+        if (!testcaseModuleNeeded.containsKey(key)) {
+            testcaseModuleNeeded.put(key, new HashSet<>());
+        }
+        return testcaseModuleNeeded.get(key);
+    }
+
+    public void setTestcaseModuleNeeded(String jobId, String jobVersion, Set<ModuleNeeded> modulesNeeded) {
+        testcaseModuleNeeded.put(getProcessKey(jobId, jobVersion), new HashSet<>(modulesNeeded));
+    }
+
     public void clearHighPriorityModuleNeeded() {
+        highPriorityModuleNeededPerJob.clear();
         highPriorityModuleNeeded.clear();
     }
 
@@ -364,10 +400,11 @@ public class LastGenerationInfo {
         pigudfNeededWithSubjobPerJob.put(key, new HashSet<String>(modulesNeeded));
     }
 
-    public void clearModulesNeededWithSubjobPerJob() {
-        if (!modulesNeededWithSubjobPerJob.isEmpty()) {
-            modulesNeededWithSubjobPerJob.clear();
-        }
+    public void clearCaches() {
+        clearHighPriorityModuleNeeded();
+        modulesNeededPerJob.clear();
+        modulesNeededWithSubjobPerJob.clear();
+        testcaseModuleNeeded.clear();
     }
 
     public void clean() {
@@ -375,7 +412,8 @@ public class LastGenerationInfo {
         routinesNeededPerJob.clear();
         pigudfNeededPerJob.clear();
         modulesNeededWithSubjobPerJob.clear();
-        highPriorityModuleNeeded.clear();
+        clearHighPriorityModuleNeeded();
+        testcaseModuleNeeded.clear();
         routinesNeededWithSubjobPerJob.clear();
         pigudfNeededWithSubjobPerJob.clear();
         contextPerJob.clear();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/projectsetting/IProjectSettingPreferenceConstants.java

@@ -76,5 +76,7 @@ public interface IProjectSettingPreferenceConstants {
      * docker profile
      */
     public static final String TEMPLATE_DOCKER_PROFILE_POM = "template_docker_profile_pom_script"; //$NON-NLS-1$
+    
+    public static final String USE_STRICT_REFERENCE_JOBLET = "use_strict_reference_joblet"; //$NON-NLS-1$
 
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/services/IGenericService.java

@@ -15,7 +15,9 @@ package org.talend.core.runtime.services;
 import java.util.List;
 import java.util.Map;
 
+import org.talend.core.GlobalServiceRegister;
 import org.talend.core.IService;
+import org.talend.core.model.components.IComponent;
 import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.INode;
 
@@ -42,4 +44,13 @@ public interface IGenericService extends IService {
 
     public void resetReferenceValue(INode curNode, String oldConnectionName, String newConnectionName);
 
+    public boolean isTcompv0(IComponent component);
+
+    public static IGenericService getService() {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericService.class)) {
+            return GlobalServiceRegister.getDefault().getService(IGenericService.class);
+        }
+        return null;
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/service/IMDMWebServiceHook.java

@@ -0,0 +1,30 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.service;
+
+import java.util.Map;
+
+import org.talend.core.IService;
+
+/**
+ * created by HHB on 2013-12-23 Detailled comment
+ *
+ */
+public interface IMDMWebServiceHook extends IService {
+
+    void preRequestSendingHook(Map<String, Object> requestContext, String userName);
+
+    String buildStudioToken(String username);
+
+    String getTokenKey();
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/service/INOSQLService.java

@@ -28,6 +28,8 @@ public interface INOSQLService extends IService {
 
     public boolean isUseReplicaSet(Connection connection);
 
+    public boolean isUseSSL(Connection connection);
+
     public String getMongoDBReplicaSets(Connection connection);
 
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/service/ITaCoKitDependencyService.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ */
+package org.talend.core.service;
+
+import java.nio.file.Path;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import org.talend.core.IService;
+import org.talend.core.model.components.IComponent;
+import org.talend.core.model.properties.Item;
+
+/**
+ * For documentation, see implementation in org.talend.sdk.component.studio-integration plugin
+ */
+public interface ITaCoKitDependencyService extends IService {
+
+    boolean hasTaCoKitComponents(final Stream<IComponent> components);
+
+    Set<String> getTaCoKitOnlyDependencies(final Stream<IComponent> components);
+
+    Stream<IComponent> getJobComponents(Item item);
+
+    Path findM2Path();
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/AbstractRoutineSynchronizer.java

@@ -19,7 +19,9 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
@@ -79,6 +81,8 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
     }
 
     private Collection<RoutineItem> getAll(ERepositoryObjectType type, boolean syncRef) throws SystemException {
+        // init code project
+        getRunProcessService().getTalendCodeJavaProject(type);
         // remove routine with same name in reference project
         final Map<String, RoutineItem> beansList = new HashMap<String, RoutineItem>();
         for (IRepositoryViewObject obj : getRepositoryService().getProxyRepositoryFactory().getAll(type)) {
@@ -95,27 +99,37 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
         return (IRepositoryService) GlobalServiceRegister.getDefault().getService(IRepositoryService.class);
     }
 
-    private void getReferencedProjectRoutine(final Map<String, RoutineItem> beansList, final Project project,
-            ERepositoryObjectType routineType, boolean syncRef) throws SystemException {
-        for (IRepositoryViewObject obj : getRepositoryService().getProxyRepositoryFactory().getAll(project, routineType)) {
+    private Set<IRepositoryViewObject> getReferencedProjectRoutine(final Map<String, RoutineItem> beansList,
+            final Project project, ERepositoryObjectType routineType, boolean syncRef) throws SystemException {
+        // init ref code project
+        if (syncRef) {
+            getRunProcessService().getTalendCodeJavaProject(routineType, project.getTechnicalLabel());
+        }
+        Set<IRepositoryViewObject> routines = new HashSet<>();
+        routines.addAll(getRepositoryService().getProxyRepositoryFactory().getAll(project, routineType));
+        for (IRepositoryViewObject obj : routines) {
             final String key = obj.getProperty().getLabel();
             // it does not have a routine with same name
             if (!beansList.containsKey(key)) {
                 beansList.put(key, (RoutineItem) obj.getProperty().getItem());
             }
-            if (syncRef) {
-                // sync routine
-                syncRoutine((RoutineItem) obj.getProperty().getItem(), false, true, true);
-            }
+        }
+        for (ProjectReference projectReference : project.getProjectReferenceList()) {
+            routines.addAll(getReferencedProjectRoutine(beansList, new Project(projectReference.getReferencedProject()),
+                    routineType, syncRef));
         }
         if (syncRef) {
+            routines.stream().forEach(obj -> {
+                try {
+                    syncRoutine((RoutineItem) obj.getProperty().getItem(), project.getTechnicalLabel(), true, true);
+                } catch (SystemException e) {
+                    ExceptionHandler.process(e);
+                }
+            });
             // sync system routine
             syncSystemRoutine(project);
         }
-
-        for (ProjectReference projectReference : project.getProjectReferenceList()) {
-            getReferencedProjectRoutine(beansList, new Project(projectReference.getReferencedProject()), routineType, syncRef);
-        }
+        return routines;
     }
 
     protected void syncSystemRoutine(Project project) throws SystemException {
@@ -128,24 +142,17 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
     }
 
     protected IFile getRoutineFile(RoutineItem routineItem) throws SystemException {
-        return getRoutineFile(routineItem, true);
+        return getRoutineFile(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel());
     }
 
-    protected IFile getRoutineFile(RoutineItem routineItem, boolean currentProject) throws SystemException {
-        String projectTechName;
-        if (currentProject) {
-            projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
-        } else {
-            projectTechName = ProjectManager.getInstance().getProject(routineItem).getTechnicalLabel();
-        }
+    protected IFile getRoutineFile(RoutineItem routineItem, String projectTechName) throws SystemException {
         ITalendProcessJavaProject talendProcessJavaProject = getRunProcessService()
                 .getTalendCodeJavaProject(ERepositoryObjectType.getItemType(routineItem), projectTechName);
         if (talendProcessJavaProject == null) {
             return null;
         }
         IFolder routineFolder = talendProcessJavaProject.getSrcSubFolder(null, routineItem.getPackageType());
-        IFile file = routineFolder.getFile(routineItem.getProperty().getLabel() + JavaUtils.JAVA_EXTENSION);
-        return file;
+        return routineFolder.getFile(routineItem.getProperty().getLabel() + JavaUtils.JAVA_EXTENSION);
     }
 
     private IFile getProcessFile(ProcessItem item) throws SystemException {
@@ -193,30 +200,31 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
 
     @Override
     public void syncRoutine(RoutineItem routineItem, boolean copyToTemp) throws SystemException {
-        syncRoutine(routineItem, true, copyToTemp, false);
+        syncRoutine(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), copyToTemp, false);
     }
 
     @Override
     public void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceUpdate) throws SystemException {
-        syncRoutine(routineItem, true, copyToTemp, forceUpdate);
+        syncRoutine(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), copyToTemp, forceUpdate);
     }
 
     @Override
-    public void syncRoutine(RoutineItem routineItem, boolean currentProject, boolean copyToTemp, boolean forceUpdate) throws SystemException {
+    public void syncRoutine(RoutineItem routineItem, String projectTechName, boolean copyToTemp, boolean forceUpdate)
+            throws SystemException {
         boolean needSync = false;
         if (routineItem != null) {
             if (forceUpdate || !isRoutineUptodate(routineItem)) {
                 needSync = true;
             } else {
-                IFile file = getRoutineFile(routineItem, currentProject);
+                IFile file = getRoutineFile(routineItem, projectTechName);
                 if (file != null && !file.exists()) {
                     needSync = true;
                 }
             }
         }
         if (needSync) {
-            doSyncRoutine(routineItem, currentProject, copyToTemp);
-            if (currentProject) {
+            doSyncRoutine(routineItem, projectTechName, copyToTemp);
+            if (ProjectManager.getInstance().getCurrentProject().getTechnicalLabel().equals(projectTechName)) {
                 setRoutineAsUptodate(routineItem);
             }
         }
@@ -224,14 +232,14 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
 
     public void syncRoutine(RoutineItem routineItem) throws SystemException {
         if (routineItem != null) {
-            doSyncRoutine(routineItem, true, true);
+            doSyncRoutine(routineItem, ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), true);
             setRoutineAsUptodate(routineItem);
         }
     }
 
-    private void doSyncRoutine(RoutineItem routineItem, boolean currentProject, boolean copyToTemp) throws SystemException {
+    private void doSyncRoutine(RoutineItem routineItem, String projectTechName, boolean copyToTemp) throws SystemException {
         try {
-            IFile file = getRoutineFile(routineItem, currentProject);
+            IFile file = getRoutineFile(routineItem, projectTechName);
             if (file == null) {
                 return;
             }
@@ -374,7 +382,7 @@ public abstract class AbstractRoutineSynchronizer implements ITalendSynchronizer
     @Override
     public void syncAllBeansForLogOn() throws SystemException {
         for (RoutineItem beanItem : getBeans(true)) {
-            syncRoutine(beanItem, true, true, true);
+            syncRoutine(beanItem, true, true);
         }
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/codegen/ITalendSynchronizer.java

@@ -45,7 +45,8 @@ public interface ITalendSynchronizer {
 
     void syncRoutine(RoutineItem routineItem, boolean copyToTemp, boolean forceUpdate) throws SystemException;
 
-    void syncRoutine(RoutineItem routineItem, boolean currentProject, boolean copyToTemp, boolean forceUpdate) throws SystemException;
+    void syncRoutine(RoutineItem routineItem, String projectTechName, boolean copyToTemp, boolean forceUpdate)
+            throws SystemException;
 
     IFile getFile(Item item) throws SystemException;
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/core/IDesignerCoreService.java

@@ -195,4 +195,7 @@ public interface IDesignerCoreService extends IService {
     public void setTACReadTimeout(int timeout);
 
     boolean isDelegateNode(INode node);
+
+    boolean isNeedContextInJar(IProcess process);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java

@@ -25,6 +25,7 @@ import org.eclipse.jdt.core.IJavaProject;
 import org.eclipse.jface.action.IAction;
 import org.eclipse.jface.preference.IPreferenceStore;
 import org.eclipse.jface.viewers.IStructuredSelection;
+import org.talend.core.GlobalServiceRegister;
 import org.talend.core.IService;
 import org.talend.core.language.ECodeLanguage;
 import org.talend.core.language.ICodeProblemsChecker;
@@ -216,6 +217,8 @@ public interface IRunProcessService extends IService {
 
     ITalendProcessJavaProject getTalendJobJavaProject(Property property);
 
+    IFolder getCodeSrcFolder(ERepositoryObjectType type, String projectTechName);
+
     ITalendProcessJavaProject getTempJavaProject();
 
     void clearProjectRelatedSettings();
@@ -237,4 +240,12 @@ public interface IRunProcessService extends IService {
     public void handleJobDependencyLoop(JobInfo mainJobInfo, List<JobInfo> listJobs, IProgressMonitor progressMonitor)
             throws Exception;
 
+    public static IRunProcessService get() {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
+            return GlobalServiceRegister.getDefault().getService(IRunProcessService.class);
+        }
+        return null;
+    }
+
+    public boolean isCIMode();
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/ProjectManager.java

@@ -21,6 +21,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang.StringUtils;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.resources.ResourcesPlugin;
@@ -91,11 +92,14 @@ public final class ProjectManager {
 
     private Set<Object> updatedRemoteHandlerRecords;
 
+    private Set<Project> tempProjects;
+
     private ProjectManager() {
         beforeLogonRecords = new HashSet<String>();
         logonRecords = new HashSet<String>();
         migrationRecords = new HashSet<String>();
         updatedRemoteHandlerRecords = new HashSet<Object>();
+        tempProjects = new HashSet<>();
         initCurrentProject();
     }
 
@@ -106,6 +110,18 @@ public final class ProjectManager {
         return singleton;
     }
 
+    public void clearTempProjects() {
+        tempProjects.clear();
+    }
+
+    public boolean removeTempProject(Project project) {
+        return tempProjects.remove(project);
+    }
+
+    public boolean addTempProject(Project project) {
+        return tempProjects.add(project);
+    }
+
     public Project getProjectFromProjectLabel(String label) {
         if (currentProject == null) {
             initCurrentProject();
@@ -119,6 +135,12 @@ public final class ProjectManager {
                 return project;
             }
         }
+        
+        for (Project project : tempProjects) {
+            if (StringUtils.equals(project.getLabel(), label)) {
+                return project;
+            }
+        }
 
         return null;
     }
@@ -136,6 +158,11 @@ public final class ProjectManager {
                 return project;
             }
         }
+        for (Project project : tempProjects) {
+            if (StringUtils.equals(project.getTechnicalLabel(), label)) {
+                return project;
+            }
+        }
 
         return null;
     }
@@ -200,6 +227,10 @@ public final class ProjectManager {
      * return all the referenced projects of current project.
      */
     public List<Project> getAllReferencedProjects(boolean force) {
+        return getAllReferencedProjects(getCurrentProject(), force);
+    }
+
+    public List<Project> getAllReferencedProjects(Project targetProject, boolean force) {
         List<Project> allReferencedprojects = new ArrayList<Project>();
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IProxyRepositoryService.class)) {
             if (this.getCurrentProject() == null) {
@@ -212,7 +243,7 @@ public final class ProjectManager {
             IProxyRepositoryFactory factory = service.getProxyRepositoryFactory();
             if (factory != null) {
                 List<org.talend.core.model.properties.Project> rProjects = factory
-                        .getReferencedProjects(this.getCurrentProject());
+                        .getReferencedProjects(targetProject);
                 if (rProjects != null) {
                     for (org.talend.core.model.properties.Project p : rProjects) {
                         Project project = new Project(p);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/model/RepositoryConstants.java

@@ -33,6 +33,8 @@ public class RepositoryConstants {
 
     public static final String PROJECT_BRANCH_ID = "repository.project.branch"; //$NON-NLS-1$
 
+    public static final String PROJECT_ID = "repository.project.id";
+
     public static final String IMG_DIRECTORY = "images"; //$NON-NLS-1$
 
     public static final String IMG_DIRECTORY_OF_JOB_OUTLINE = "images/job_outlines"; //$NON-NLS-1$
@@ -98,6 +100,8 @@ public class RepositoryConstants {
 
     public static final String REPOSITORY_CLOUD_APAC_ID = "cloud_apac"; //$NON-NLS-1$
 
+    public static final String REPOSITORY_CLOUD_US_WEST_ID = "cloud_us_west"; //$NON-NLS-1$
+
     public static final String REPOSITORY_CLOUD_CUSTOM_ID = "cloud_custom"; //$NON-NLS-1$
 
     public static final String REPOSITORY_URL = "url"; //$NON-NLS-1$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/ui/login/connections/EncryptedProperties.java

@@ -14,11 +14,9 @@ package org.talend.repository.ui.login.connections;
 
 import java.util.Properties;
 
-import org.talend.commons.ui.runtime.exception.ExceptionHandler;
-import org.talend.daikon.crypto.EncodingUtils;
-import org.talend.daikon.crypto.KeySources;
 import org.talend.daikon.security.CryptoHelper;
 
+
 /**
  * DOC hwang class global comment. Detailled comment
  */
@@ -27,16 +25,7 @@ public class EncryptedProperties extends Properties {
     private CryptoHelper crypto;
 
     public EncryptedProperties() {
-        String key = System.getProperty("properties.encryption.key");
-        if (key == null) {
-            try {
-                byte[] byteKey = KeySources.file("properties.encryption.key").getKey();
-                key = new String(byteKey, EncodingUtils.ENCODING);
-            } catch (Exception ex) {
-                ExceptionHandler.process(ex);
-            }
-        }
-        crypto = new CryptoHelper(key);
+        crypto = new CryptoHelper("Il faudrait trouver une passphrase plus originale que celle-ci!");
     }
 
     public String getProperty(String key) {

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/utils/PluginUtil.java

@@ -12,11 +12,18 @@
 // ============================================================================
 package org.talend.core.ui.utils;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
 import java.net.URL;
+import java.util.Properties;
 
 import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.URIUtil;
 import org.eclipse.ui.IEditorPart;
 import org.eclipse.ui.IWorkbenchPage;
 import org.eclipse.ui.IWorkbenchWindow;
@@ -104,4 +111,28 @@ public class PluginUtil {
         }
         return part.getEditorSite().getId();
     }
+
+    public static File getStudioConfigFile() throws Exception {
+        URL configLocation = new URL("platform:/config/config.ini"); //$NON-NLS-1$
+        URL fileUrl = FileLocator.toFileURL(configLocation);
+        return URIUtil.toFile(new URI(fileUrl.getProtocol(), fileUrl.getPath(), fileUrl.getQuery()));
+    }
+
+    public static Properties readProperties(final File config) {
+        final Properties configuration = new Properties();
+        try (final InputStream stream = new FileInputStream(config)) {
+            configuration.load(stream);
+        } catch (final IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+        return configuration;
+    }
+
+    public static void saveProperties(final File config, Properties prop, String comment) {
+        try (FileOutputStream oFile = new FileOutputStream(config)) {
+            prop.store(oFile, comment);
+        } catch (final IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
 }

main/plugins/org.talend.core/src/main/java/org/talend/core/CoreService.java

@@ -54,6 +54,7 @@ import org.talend.core.model.metadata.QueryUtil;
 import org.talend.core.model.metadata.builder.ConvertionHelper;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
 import org.talend.core.model.process.ElementParameterParser;
+import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.model.repository.IRepositoryViewObject;
@@ -280,9 +281,10 @@ public class CoreService implements ICoreService {
             ICodeGeneratorService codeGenService = (ICodeGeneratorService) GlobalServiceRegister.getDefault().getService(
                     ICodeGeneratorService.class);
             codeGenService.createRoutineSynchronizer().syncAllRoutinesForLogOn();
-            codeGenService.createRoutineSynchronizer().syncAllPigudfForLogOn();
+            if (ProcessUtils.isRequiredPigUDFs(null)) {
+                codeGenService.createRoutineSynchronizer().syncAllPigudfForLogOn();
+            }
         }
-
     }
 
     @Override

main/plugins/org.talend.core/src/main/java/org/talend/core/model/components/conversions/RenameComponentConversion.java

@@ -43,7 +43,7 @@ public class RenameComponentConversion implements IComponentConversion {
         replaceAllInAllNodesParameterValue(item, oldNodeUniqueName, newNodeUniqueName);
     }
 
-    private static void replaceAllInAllNodesParameterValue(ProcessType item, String oldName, String newName) {
+    protected static void replaceAllInAllNodesParameterValue(ProcessType item, String oldName, String newName) {
         for (Object o : item.getNode()) {
             NodeType nt = (NodeType) o;
             ComponentUtilities.replaceInNodeParameterValue(nt, oldName, newName);

main/plugins/org.talend.core/src/main/java/org/talend/core/model/components/conversions/RenameETLJDBCComponentConversion.java

@@ -0,0 +1,40 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.components.conversions;
+
+import org.talend.core.model.components.ComponentUtilities;
+import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
+import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
+
+public class RenameETLJDBCComponentConversion extends RenameComponentConversion{
+
+    private String newName;
+    
+    private String oldName;
+
+    public RenameETLJDBCComponentConversion(String newName, String oldName) {
+        super(newName);
+        this.newName = newName;
+        this.oldName = oldName;
+    }
+
+    public void transform(NodeType node) {
+        node.setComponentName(newName);
+        ProcessType item = (ProcessType) node.eContainer();
+        String oldNodeUniqueName = ComponentUtilities.getNodeUniqueName(node);
+        String newNodeUniqueName = oldNodeUniqueName.replaceAll(oldName, newName);
+        ComponentUtilities.setNodeUniqueName(node, newNodeUniqueName);
+        replaceAllInAllNodesParameterValue(item, oldNodeUniqueName, newNodeUniqueName);
+    }
+
+}

main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java

@@ -64,6 +64,8 @@ import org.talend.core.language.LanguageManager;
 import org.talend.core.model.components.ComponentCategory;
 import org.talend.core.model.components.EComponentType;
 import org.talend.core.model.components.IComponent;
+import org.talend.core.model.components.IComponentsFactory;
+import org.talend.core.model.components.IComponentsService;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.Project;
 import org.talend.core.model.metadata.IMetadataColumn;
@@ -81,6 +83,7 @@ import org.talend.core.model.process.JobInfo;
 import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.process.ReplaceNodesInProcessProvider;
 import org.talend.core.model.properties.Item;
+import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.relationship.Relation;
@@ -92,10 +95,12 @@ import org.talend.core.model.repository.job.JobResource;
 import org.talend.core.model.repository.job.JobResourceManager;
 import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.repository.model.ProxyRepositoryFactory;
+import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.runtime.process.ITalendProcessJavaProject;
 import org.talend.core.runtime.process.LastGenerationInfo;
 import org.talend.core.runtime.process.TalendProcessArgumentConstant;
 import org.talend.core.runtime.process.TalendProcessOptionConstants;
+import org.talend.core.runtime.projectsetting.ProjectPreferenceManager;
 import org.talend.core.runtime.repository.build.BuildExportManager;
 import org.talend.core.service.IResourcesDependenciesService;
 import org.talend.core.services.ICoreTisService;
@@ -152,7 +157,7 @@ public class ProcessorUtilities {
     private static boolean exportJobAsMicroService = false;
 
     private static IDesignerCoreService designerCoreService =
-            (IDesignerCoreService) GlobalServiceRegister.getDefault().getService(IDesignerCoreService.class);
+            GlobalServiceRegister.getDefault().getService(IDesignerCoreService.class);
 
     private static Map<String, Integer> lastGeneratedWithStatsOrTrace = new HashMap<String, Integer>();
 
@@ -170,6 +175,8 @@ public class ProcessorUtilities {
 
     private static boolean isDebug = false;
 
+    private static boolean isCIMode = false;
+
     private static JobInfo mainJobInfo;
 
     public static void addOpenEditor(IEditorPart editor) {
@@ -557,7 +564,10 @@ public class ProcessorUtilities {
                     neededLibraries);
 
             // get all job testcases needed modules
-            neededLibraries.addAll(getAllJobTestcaseModules(selectedProcessItem));
+            Set<ModuleNeeded> testcaseModules = getAllJobTestcaseModules(selectedProcessItem);
+            LastGenerationInfo.getInstance().setTestcaseModuleNeeded(jobInfo.getJobId(), jobInfo.getJobVersion(),
+                    testcaseModules);
+            neededLibraries.addAll(testcaseModules);
 
             // must install the needed libraries before generate codes with poms.
             CorePlugin.getDefault().getRunProcessService().updateLibraries(neededLibraries, currentProcess,
@@ -613,28 +623,55 @@ public class ProcessorUtilities {
         return processor;
     }
 
-    private static boolean checkLoopDependencies(Relation mainJobInfo, Map<String, String> idToLastestVersionMap)
+    public static boolean checkLoopDependencies(Relation mainJobInfo, Map<String, String> idToLastestVersionMap)
             throws ProcessorException {
         List<Relation> itemsJobRelatedTo = getItemsRelation(mainJobInfo, idToLastestVersionMap);
         List<Relation> relationChecked = new ArrayList<>();
         relationChecked.add(mainJobInfo);
-        return checkLoopDependencies(mainJobInfo, itemsJobRelatedTo, relationChecked, idToLastestVersionMap);
+        return checkLoopDependencies(mainJobInfo, mainJobInfo, itemsJobRelatedTo, relationChecked, idToLastestVersionMap);
     }
 
-    private static boolean checkLoopDependencies(Relation mainRelation, List<Relation> itemsJobRelatedTo,
+    private static boolean checkLoopDependencies(Relation mainRelation, Relation currentRelation,
+            List<Relation> itemsJobRelatedTo,
             List<Relation> relationChecked, Map<String, String> idToLastestVersionMap) throws ProcessorException {
         boolean hasDependency = false;
         for (Relation relation : itemsJobRelatedTo) {
+            try {
+                // means the tRunjob deactivate, or one of the specific version tRunjon deactivate, skip
+                Map<String, Set<String>> actTrunjobHM = getActivateTRunjobMap(currentRelation.getId(),
+                        currentRelation.getVersion());
+                if (actTrunjobHM.get(relation.getId()) == null
+                        || !actTrunjobHM.get(relation.getId()).contains(relation.getVersion())) {
+                    continue;
+                }
+            } catch (Exception e) {
+                throw new ProcessorException(e);
+            }
+
             hasDependency = relation.getId().equals(mainRelation.getId())
                     && relation.getVersion().equals(mainRelation.getVersion());
             if (!hasDependency) {
                 List<Relation> itemsChildJob = getItemsRelation(relation, idToLastestVersionMap);
                 if (!relationChecked.contains(relation)) {
                     relationChecked.add(relation);
-                    hasDependency = checkLoopDependencies(mainRelation, itemsChildJob, relationChecked, idToLastestVersionMap);
+                    hasDependency = checkLoopDependencies(mainRelation, relation, itemsChildJob, relationChecked,
+                            idToLastestVersionMap);
                 }
                 if (!hasDependency) {
                     for (Relation childRelation : itemsChildJob) {
+
+                        try {
+                            // means the tRunjob deactivate, or one of the specific version tRunjon deactivate, skip
+                            Map<String, Set<String>> activateTRunjobMap = getActivateTRunjobMap(relation.getId(),
+                                    relation.getVersion());
+                            if (activateTRunjobMap.get(childRelation.getId()) == null
+                                    || !activateTRunjobMap.get(childRelation.getId()).contains(childRelation.getVersion())) {
+                                continue;
+                            }
+                        } catch (Exception e) {
+                            throw new ProcessorException(e);
+                        }
+
                         hasDependency = checkLoopDependencies(childRelation, idToLastestVersionMap);
                         if (hasDependency) {
                             break;
@@ -650,6 +687,133 @@ public class ProcessorUtilities {
         return hasDependency;
     }
 
+    private static Map<String, Set<String>> getActivateTRunjobMap(String id, String version) throws PersistenceException {
+        Map<String, Set<String>> actTrunjobHM = new HashMap<String, Set<String>>();
+        ProcessType processType = null;
+        try {
+            IRepositoryViewObject currentJobObject = ProxyRepositoryFactory.getInstance().getSpecificVersion(id, version, true);
+            if (currentJobObject != null) {
+                Item item = currentJobObject.getProperty().getItem();
+                if (item instanceof ProcessItem) {
+                    processType = ((ProcessItem) item).getProcess();
+                } else if (item instanceof JobletProcessItem) {
+                    processType = ((JobletProcessItem) item).getJobletProcess();
+                }
+            }
+        } catch (PersistenceException e) {
+            ExceptionHandler.process(e);
+        }
+        if (processType != null) {
+            List<Project> allProjects = new ArrayList<Project>();
+            allProjects.add(ProjectManager.getInstance().getCurrentProject());
+            allProjects.addAll(ProjectManager.getInstance().getAllReferencedProjects());
+
+            List<String> jobletsComponentsList = new ArrayList<String>();
+            IComponentsFactory componentsFactory = null;
+            if (GlobalServiceRegister.getDefault().isServiceRegistered(IComponentsService.class)) {
+                IComponentsService compService = GlobalServiceRegister.getDefault()
+                        .getService(IComponentsService.class);
+                if (compService != null) {
+                    componentsFactory = compService.getComponentsFactory();
+                    for (IComponent component : componentsFactory.readComponents()) {
+                        if (component.getComponentType() == EComponentType.JOBLET) {
+                            jobletsComponentsList.add(component.getName());
+                        }
+                    }
+                }
+            }
+
+            String jobletPaletteType = null;
+            String frameWork = processType.getFramework();
+            if (StringUtils.isBlank(frameWork)) {
+                jobletPaletteType = ComponentCategory.CATEGORY_4_DI.getName();
+            } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK)) {
+                jobletPaletteType = ComponentCategory.CATEGORY_4_SPARK.getName();
+            } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK_STREAMING)) {
+                jobletPaletteType = ComponentCategory.CATEGORY_4_SPARKSTREAMING.getName();
+            }
+
+            for (Object nodeObject : processType.getNode()) {
+                NodeType node = (NodeType) nodeObject;
+                // not tRunjob && not joblet then continue
+                if (!node.getComponentName().equals("tRunJob") && !jobletsComponentsList.contains(node.getComponentName())) { // $NON-NLS-1$
+                    continue;
+                }
+                boolean nodeActivate = true;
+                String processIds = null;
+                String processVersion = null;
+                for (Object elementParam : node.getElementParameter()) {
+                    ElementParameterType elemParamType = (ElementParameterType) elementParam;
+                    if ("PROCESS:PROCESS_TYPE_PROCESS".equals(elemParamType.getName())) { // $NON-NLS-1$
+                        processIds = elemParamType.getValue();
+                        if (StringUtils.isNotBlank(processIds)) {
+                            for (String jobId : processIds.split(ProcessorUtilities.COMMA)) {
+                                if (actTrunjobHM.get(jobId) == null) {
+                                    actTrunjobHM.put(jobId, new HashSet<String>());
+                                }
+                            }
+                        }
+                    } else if ("PROCESS:PROCESS_TYPE_VERSION".equals(elemParamType.getName()) // $NON-NLS-1$
+                            || "PROCESS_TYPE_VERSION".equals(elemParamType.getName())) { // $NON-NLS-1$
+                        processVersion = elemParamType.getValue();
+                    } else if ("ACTIVATE".equals(elemParamType.getName())) { // $NON-NLS-1$
+                        nodeActivate = Boolean.parseBoolean(elemParamType.getValue());
+                    }
+                }
+
+                if (nodeActivate) {
+                    if (StringUtils.isNotBlank(processIds)) {
+                        for (String jobId : processIds.split(ProcessorUtilities.COMMA)) {
+                            String actualVersion = processVersion;
+                            if (RelationshipItemBuilder.LATEST_VERSION.equals(processVersion)) {
+                                for (Project project : allProjects) {
+                                    IRepositoryViewObject lastVersion = null;
+                                    lastVersion = ProxyRepositoryFactory.getInstance().getLastVersion(project, jobId);
+                                    if (lastVersion != null) {
+                                        actualVersion = lastVersion.getVersion();
+                                        break;
+                                    }
+                                }
+                            }
+                            if (actTrunjobHM.get(jobId) != null) {
+                                actTrunjobHM.get(jobId).add(actualVersion);
+                            }
+
+                        }
+                    } else if (componentsFactory != null && jobletPaletteType != null) {
+                        // for joblet
+                        IComponent cc = componentsFactory.get(node.getComponentName(), jobletPaletteType);
+                        if (GlobalServiceRegister.getDefault().isServiceRegistered(IJobletProviderService.class)) {
+                            IJobletProviderService jobletService = GlobalServiceRegister.getDefault()
+                                    .getService(IJobletProviderService.class);
+                            Property property = jobletService.getJobletComponentItem(cc);
+                            if (property != null && StringUtils.isNotBlank(property.getId())) {
+                                String jobletId = property.getId();
+                                if (actTrunjobHM.get(jobletId) == null) {
+                                    actTrunjobHM.put(jobletId, new HashSet<String>());
+                                }
+                                String actualVersion = processVersion;
+                                if (RelationshipItemBuilder.LATEST_VERSION.equals(processVersion)) {
+                                    for (Project project : allProjects) {
+                                        IRepositoryViewObject lastVersion = null;
+                                        lastVersion = ProxyRepositoryFactory.getInstance().getLastVersion(project, jobletId);
+                                        if (lastVersion != null) {
+                                            actualVersion = lastVersion.getVersion();
+                                            break;
+                                        }
+                                    }
+                                }
+
+                                actTrunjobHM.get(jobletId).add(actualVersion);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return actTrunjobHM;
+    }
+
     private static List<Relation> getItemsRelation(Relation mainJobInfo, Map<String, String> idToLastestVersionMap) throws ProcessorException {
         List<Relation> itemsJobRelatedTo = new ArrayList<Relation>();
         try {
@@ -733,11 +897,18 @@ public class ProcessorUtilities {
 
                     IFolder xmlMappingFolder = jobInfo.getProcessor().getTalendJavaProject().getResourceSubFolder(null,
                             JavaUtils.JAVA_XML_MAPPING);
-                    if (xmlMappingFolder.members().length == 0
+                    ProjectPreferenceManager manager = CoreRuntimePlugin.getInstance().getProjectPreferenceManager();
+                    boolean updated = manager.getBoolean(MetadataTalendType.UPDATED_MAPPING_FILES);
+                    if ((xmlMappingFolder.members().length == 0 || updated)
                             && GlobalServiceRegister.getDefault().isServiceRegistered(ICoreService.class)) {
                         ICoreService coreService =
-                                (ICoreService) GlobalServiceRegister.getDefault().getService(ICoreService.class);
+                                GlobalServiceRegister.getDefault().getService(ICoreService.class);
                         coreService.synchronizeMapptingXML(jobInfo.getProcessor().getTalendJavaProject());
+                        // reset
+                        if (updated) {
+                            manager.setValue(MetadataTalendType.UPDATED_MAPPING_FILES, false);
+                            manager.save();
+                        }
                     }
                 }
             } catch (Exception e) {
@@ -758,7 +929,7 @@ public class ProcessorUtilities {
         boolean hasDynamicMetadata = false;
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IDesignerCoreService.class)) {
             IDesignerCoreService designerCoreService =
-                    (IDesignerCoreService) GlobalServiceRegister.getDefault().getService(IDesignerCoreService.class);
+                    GlobalServiceRegister.getDefault().getService(IDesignerCoreService.class);
             for (INode node : currentProcess.getGraphicalNodes()) {
                 if (designerCoreService.isDelegateNode(node)) { // for jdbc, currently
                     return true;
@@ -768,7 +939,7 @@ public class ProcessorUtilities {
 
         ICoreTisService service = null;
         if (GlobalServiceRegister.getDefault().isServiceRegistered(ICoreTisService.class)) {
-            service = (ICoreTisService) GlobalServiceRegister.getDefault().getService(ICoreTisService.class);
+            service = GlobalServiceRegister.getDefault().getService(ICoreTisService.class);
         }
         for (INode node : (List<? extends INode>) currentProcess.getGeneratingNodes()) {
             if (node.getComponent() != null && node.getComponent().getComponentType() == EComponentType.GENERIC) {
@@ -941,8 +1112,8 @@ public class ProcessorUtilities {
     private static void generateDataSet(IProcess process, IProcessor processor) {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
             ITestContainerProviderService testContainerService =
-                    (ITestContainerProviderService) GlobalServiceRegister.getDefault().getService(
-                            ITestContainerProviderService.class);
+                    GlobalServiceRegister.getDefault().getService(
+                    ITestContainerProviderService.class);
             if (testContainerService != null) {
                 if (!testContainerService.isTestContainerProcess(process)) {
                     return;
@@ -1116,7 +1287,7 @@ public class ProcessorUtilities {
 
             Set<ModuleNeeded> neededLibraries =
                     CorePlugin.getDefault().getDesignerCoreService().getNeededLibrariesForProcess(currentProcess,
-                            false);
+                            isCIMode && BitwiseOptionUtils.containOption(option, GENERATE_MAIN_ONLY));
             if (neededLibraries != null) {
                 LastGenerationInfo.getInstance().setModulesNeededWithSubjobPerJob(jobInfo.getJobId(),
                         jobInfo.getJobVersion(), neededLibraries);
@@ -1124,7 +1295,10 @@ public class ProcessorUtilities {
                         neededLibraries);
 
                 // get all job testcases needed modules
-                neededLibraries.addAll(getAllJobTestcaseModules(selectedProcessItem));
+                Set<ModuleNeeded> testcaseModules = getAllJobTestcaseModules(selectedProcessItem);
+                LastGenerationInfo.getInstance().setTestcaseModuleNeeded(jobInfo.getJobId(), jobInfo.getJobVersion(),
+                        testcaseModules);
+                neededLibraries.addAll(testcaseModules);
 
                 // must install the needed libraries before generate codes with poms.
                 CorePlugin.getDefault().getRunProcessService().updateLibraries(neededLibraries, currentProcess,
@@ -1206,8 +1380,8 @@ public class ProcessorUtilities {
         Set<ModuleNeeded> neededLibraries = new HashSet<>();
         if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
             ITestContainerProviderService testcontainerService =
-                    (ITestContainerProviderService) GlobalServiceRegister.getDefault().getService(
-                            ITestContainerProviderService.class);
+                    GlobalServiceRegister.getDefault().getService(
+                    ITestContainerProviderService.class);
             if (!testcontainerService.isTestContainerItem(selectedProcessItem)) {
                 try {
                     neededLibraries.addAll(testcontainerService.getAllJobTestcaseModules(selectedProcessItem));
@@ -1229,14 +1403,23 @@ public class ProcessorUtilities {
     public static void cleanSourceFolder(IProgressMonitor progressMonitor, IProcess currentProcess,
             IProcessor processor) {
         try {
+            ITalendProcessJavaProject jobProject = processor.getTalendJavaProject();
+            // clean up source code
             IPath codePath = processor.getSrcCodePath().removeLastSegments(2);
-            IFolder srcFolder = processor.getTalendJavaProject().getProject().getFolder(codePath);
+            IFolder srcFolder = jobProject.getProject().getFolder(codePath);
             String jobPackageFolder = JavaResourcesHelper.getJobClassPackageFolder(currentProcess);
             for (IResource resource : srcFolder.members()) {
-                if (resource.getProjectRelativePath().toPortableString().endsWith(jobPackageFolder)) {
-                    break;
+                if (!resource.getProjectRelativePath().toPortableString().endsWith(jobPackageFolder)) {
+                    resource.delete(true, progressMonitor);
+                }
+            }
+            // clean up resources folder if needed
+            if (ProcessorUtilities.isExportConfig() && !designerCoreService.isNeedContextInJar(currentProcess)) {
+                for (IResource resource : jobProject.getResourcesFolder().members()) {
+                    if (resource.exists()) {
+                        resource.delete(true, progressMonitor);
+                    }
                 }
-                resource.delete(true, progressMonitor);
             }
         } catch (CoreException e) {
             ExceptionHandler.process(e);
@@ -1279,7 +1462,7 @@ public class ProcessorUtilities {
         // 2.TDQ-14308 current drools file in 'src/resourcesmetadata/survivorship/' should be included to job jar.
         if (GlobalServiceRegister.getDefault().isServiceRegistered(ITDQItemService.class)) {
             ITDQItemService tdqItemService =
-                    (ITDQItemService) GlobalServiceRegister.getDefault().getService(ITDQItemService.class);
+                    GlobalServiceRegister.getDefault().getService(ITDQItemService.class);
             if (tdqItemService == null) {
                 return;
             }
@@ -1333,7 +1516,7 @@ public class ProcessorUtilities {
         }
         IProcess2 process = (IProcess2) currentProcess;
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IResourcesDependenciesService.class)) {
-            IResourcesDependenciesService resourcesService = (IResourcesDependenciesService) GlobalServiceRegister.getDefault()
+            IResourcesDependenciesService resourcesService = GlobalServiceRegister.getDefault()
                     .getService(IResourcesDependenciesService.class);
             if (resourcesService == null) {
                 return;
@@ -1410,8 +1593,8 @@ public class ProcessorUtilities {
         if (BitwiseOptionUtils.containOption(option, GENERATE_TESTS) && jobInfo.getProcessItem() != null) {
             if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
                 ITestContainerProviderService testContainerService =
-                        (ITestContainerProviderService) GlobalServiceRegister.getDefault().getService(
-                                ITestContainerProviderService.class);
+                        GlobalServiceRegister.getDefault().getService(
+                        ITestContainerProviderService.class);
                 if (testContainerService != null) {
                     List<ProcessItem> testsItems =
                             testContainerService.getTestContainersByVersion(jobInfo.getProcessItem());
@@ -1821,7 +2004,7 @@ public class ProcessorUtilities {
             boolean contextProperties, boolean applyToChildren) throws ProcessorException {
         ISVNProviderService service = null;
         if (PluginChecker.isSVNProviderPluginLoaded()) {
-            service = (ISVNProviderService) GlobalServiceRegister.getDefault().getService(ISVNProviderService.class);
+            service = GlobalServiceRegister.getDefault().getService(ISVNProviderService.class);
         }
         if (service != null && service.isProjectInSvnMode()) {
             RepositoryManager.syncRoutineAndJoblet(ERepositoryObjectType.ROUTINES);
@@ -1873,7 +2056,7 @@ public class ProcessorUtilities {
         // added by nma, to refresh routines when generating code in SVN mode. 10225.
         ISVNProviderService service = null;
         if (PluginChecker.isSVNProviderPluginLoaded()) {
-            service = (ISVNProviderService) GlobalServiceRegister.getDefault().getService(ISVNProviderService.class);
+            service = GlobalServiceRegister.getDefault().getService(ISVNProviderService.class);
         }
         if (service != null && service.isProjectInSvnMode()) {
             RepositoryManager.syncRoutineAndJoblet(ERepositoryObjectType.ROUTINES);
@@ -1910,7 +2093,7 @@ public class ProcessorUtilities {
 
         ISVNProviderService service = null;
         if (PluginChecker.isSVNProviderPluginLoaded()) {
-            service = (ISVNProviderService) GlobalServiceRegister.getDefault().getService(ISVNProviderService.class);
+            service = GlobalServiceRegister.getDefault().getService(ISVNProviderService.class);
         }
         if (service != null && service.isProjectInSvnMode()) {
             RepositoryManager.syncRoutineAndJoblet(ERepositoryObjectType.ROUTINES);
@@ -1968,7 +2151,7 @@ public class ProcessorUtilities {
         // added by nma, to refresh routines when generating code in SVN mode. 10225.
         ISVNProviderService service = null;
         if (PluginChecker.isSVNProviderPluginLoaded()) {
-            service = (ISVNProviderService) GlobalServiceRegister.getDefault().getService(ISVNProviderService.class);
+            service = GlobalServiceRegister.getDefault().getService(ISVNProviderService.class);
         }
         if (service != null && service.isProjectInSvnMode()) {
             RepositoryManager.syncRoutineAndJoblet(ERepositoryObjectType.ROUTINES);
@@ -2247,8 +2430,8 @@ public class ProcessorUtilities {
         if (parentJobInfo.isTestContainer()
                 && GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
             ITestContainerProviderService testContainerService =
-                    (ITestContainerProviderService) GlobalServiceRegister.getDefault().getService(
-                            ITestContainerProviderService.class);
+                    GlobalServiceRegister.getDefault().getService(
+                    ITestContainerProviderService.class);
             if (testContainerService != null) {
             	getSubjobInfo(testContainerService.getOriginalNodes(ptype), ptype, parentJobInfo, jobInfos,firstChildOnly);
             }
@@ -2257,8 +2440,8 @@ public class ProcessorUtilities {
         if (!parentJobInfo.isTestContainer() && !parentJobInfo.isJoblet()
                 && GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
             ITestContainerProviderService testContainerService =
-                    (ITestContainerProviderService) GlobalServiceRegister.getDefault().getService(
-                            ITestContainerProviderService.class);
+                    GlobalServiceRegister.getDefault().getService(
+                    ITestContainerProviderService.class);
             if (testContainerService != null) {
                 List<ProcessItem> testsItems =
                         testContainerService.getTestContainersByVersion(parentJobInfo.getProcessItem());
@@ -2345,8 +2528,8 @@ public class ProcessorUtilities {
                 // for joblet node
                 if (jobletPaletteType != null && PluginChecker.isJobLetPluginLoaded()) {
                     IJobletProviderService service =
-                            (IJobletProviderService) GlobalServiceRegister.getDefault().getService(
-                                    IJobletProviderService.class);
+                            GlobalServiceRegister.getDefault().getService(
+                            IJobletProviderService.class);
                     if (service != null) {
                         IComponent jobletComponent = service.getJobletComponent(node, jobletPaletteType);
                         ProcessType jobletProcess = service.getJobletProcess(jobletComponent);
@@ -2420,8 +2603,8 @@ public class ProcessorUtilities {
         } else {
             if (GlobalServiceRegister.getDefault().isServiceRegistered(IJobletProviderService.class)) {
                 IJobletProviderService jobletService =
-                        (IJobletProviderService) GlobalServiceRegister.getDefault().getService(
-                                IJobletProviderService.class);
+                        GlobalServiceRegister.getDefault().getService(
+                        IJobletProviderService.class);
                 if (jobletService.isJobletItem(processItem)) {
                     processType = jobletService.getJobletProcess(processItem);
                     parentJobInfo = new JobInfo(processItem.getProperty(), processType.getDefaultContext());
@@ -2430,8 +2613,8 @@ public class ProcessorUtilities {
         }
         if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
             ITestContainerProviderService testContainerService =
-                    (ITestContainerProviderService) GlobalServiceRegister.getDefault().getService(
-                            ITestContainerProviderService.class);
+                    GlobalServiceRegister.getDefault().getService(
+                    ITestContainerProviderService.class);
             if (testContainerService.isTestContainerItem(processItem)) {
                 parentJobInfo.setTestContainer(true);
             }
@@ -2508,7 +2691,7 @@ public class ProcessorUtilities {
     public static File getJavaProjectLibFolder() {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
             IRunProcessService processService =
-                    (IRunProcessService) GlobalServiceRegister.getDefault().getService(IRunProcessService.class);
+                    GlobalServiceRegister.getDefault().getService(IRunProcessService.class);
             return processService.getJavaProjectLibFolder().getLocation().toFile();
         }
         return null;
@@ -2517,7 +2700,7 @@ public class ProcessorUtilities {
     public static String getJavaProjectLibFolderPath() {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
             IRunProcessService processService =
-                    (IRunProcessService) GlobalServiceRegister.getDefault().getService(IRunProcessService.class);
+                    GlobalServiceRegister.getDefault().getService(IRunProcessService.class);
             return processService.getJavaProjectLibFolder().getLocation().toPortableString();
         }
         return null;
@@ -2589,7 +2772,54 @@ public class ProcessorUtilities {
     }
 
     public static boolean isEsbJob(String processId, String version) {
-        return esbJobs.contains(esbJobKey(processId, version));
+        ProcessItem processItem = ItemCacheManager.getProcessItem(processId, version);
+        if (processItem != null && processItem.getProperty() != null && processItem.getProperty().getItem() != null) {
+            Set<JobInfo> infos = ProcessorUtilities.getChildrenJobInfo(processItem.getProperty().getItem(), false);
+            for (JobInfo jobInfo : infos) {
+                ProcessType processType = jobInfo.getProcessItem().getProcess();
+                EList<NodeType> nodes = processType.getNode();
+                for (NodeType nodeType : nodes) {
+                    if (isEsbComponentName(nodeType.getComponentName())) {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }
+        return false;
+    }
+    
+    public static boolean isEsbJob(IProcess process) {
+        return isEsbJob(process, false);
+    }
+
+    public static boolean isEsbJob(IProcess process, boolean checkCurrentProcess) {
+
+        if (process instanceof IProcess2) {
+
+            if (checkCurrentProcess) {
+                for (INode n : process.getGraphicalNodes()) {
+                    if (isEsbComponentName(n.getComponent().getName())) {
+                        return true;
+                    }
+                }
+            } else {
+                Set<JobInfo> infos = ProcessorUtilities.getChildrenJobInfo(((IProcess2) process).getProperty().getItem(), false);
+
+                for (JobInfo jobInfo : infos) {
+                    ProcessType processType = jobInfo.getProcessItem().getProcess();
+                    EList<NodeType> nodes = processType.getNode();
+                    for (NodeType nodeType : nodes) {
+                        if (isEsbComponentName(nodeType.getComponentName())) {
+                            return true;
+                        }
+                    }
+                }
+            }
+            return false;
+        }
+
+        return false;
     }
 
     private static void addEsbJob(JobInfo jobInfo) {
@@ -2629,4 +2859,12 @@ public class ProcessorUtilities {
         return "tRunJob".equalsIgnoreCase(componentName) || "cTalendJob".equalsIgnoreCase(componentName);
     }
 
+    public static boolean isCIMode() {
+        return isCIMode;
+    }
+
+    public static void setCIMode(boolean isCIMode) {
+        ProcessorUtilities.isCIMode = isCIMode;
+    }
+
 }

main/plugins/org.talend.datatools.xml/src/main/java/org/talend/datatools/xml/utils/ATreeNode.java

@@ -277,6 +277,9 @@ public class ATreeNode {
      * @throws OdaException
      */
     public void setDataType(String type) throws OdaException {
+        if (type == null) {
+            type = "";
+        }
         originalDataType = new String(type);
         this.dataType = getDataType(type);
     }

main/plugins/org.talend.datatools.xml/src/main/java/org/talend/datatools/xml/utils/XSDPopulationUtil2.java

@@ -43,8 +43,10 @@ import org.eclipse.xsd.XSDImport;
 import org.eclipse.xsd.XSDModelGroup;
 import org.eclipse.xsd.XSDParticle;
 import org.eclipse.xsd.XSDSchema;
+import org.eclipse.xsd.XSDSimpleTypeDefinition;
 import org.eclipse.xsd.XSDTerm;
 import org.eclipse.xsd.XSDTypeDefinition;
+import org.eclipse.xsd.XSDVariety;
 import org.eclipse.xsd.impl.XSDNamedComponentImpl;
 import org.eclipse.xsd.util.XSDConstants;
 import org.eclipse.xsd.util.XSDResourceImpl;
@@ -355,12 +357,15 @@ public class XSDPopulationUtil2 implements IXSDPopulationUtil {
                 }
             }
             if (!resolvedAsComplex) {
-                String dataType = xsdElementDeclarationParticle.getTypeDefinition().getQName();
+                XSDTypeDefinition typeDefinition = xsdElementDeclarationParticle.getTypeDefinition();
+                String dataType = typeDefinition.getQName();
                 if (!XSDConstants
-                        .isSchemaForSchemaNamespace(xsdElementDeclarationParticle.getTypeDefinition().getTargetNamespace())
-                        && xsdElementDeclarationParticle.getTypeDefinition().getBaseType() != null) {
-                    if (!"xs:anySimpleType".equals(xsdElementDeclarationParticle.getTypeDefinition().getBaseType().getQName())) {
-                        dataType = xsdElementDeclarationParticle.getTypeDefinition().getBaseType().getQName();
+                        .isSchemaForSchemaNamespace(typeDefinition.getTargetNamespace())
+                        && typeDefinition.getBaseType() != null) {
+                    if (!"xs:anySimpleType".equals(typeDefinition.getBaseType().getQName())) {
+                        dataType = typeDefinition.getBaseType().getQName();
+                    } else if (typeDefinition instanceof XSDSimpleTypeDefinition) {
+                        dataType = getVarietyType(((XSDSimpleTypeDefinition) typeDefinition).getVariety());
                     }
                 }
                 partNode.setDataType(dataType);
@@ -378,6 +383,16 @@ public class XSDPopulationUtil2 implements IXSDPopulationUtil {
         }
     }
 
+    private String getVarietyType(XSDVariety variety) {
+        String dataType = "xs:anyType";
+        if (XSDVariety.LIST_LITERAL.equals(variety)) {
+            dataType = "xs:list";
+        } else if (XSDVariety.UNION_LITERAL.equals(variety)) {
+            dataType = "xs:union";
+        }
+        return dataType;
+    }
+
     private void handleOptionalAttribute(ATreeNode node, XSDParticle xsdParticle) {
         if (node == null || xsdParticle == null || xsdParticle.getElement() == null) {
             return;
@@ -485,6 +500,17 @@ public class XSDPopulationUtil2 implements IXSDPopulationUtil {
                 }
                 if (xsdTypeDefinition instanceof XSDComplexTypeDefinition) {
                     addComplexTypeDetails(xsdSchema, node, xsdTypeDefinition, prefix, namespace, "/" + elementName + "/");
+                } else if (xsdTypeDefinition instanceof XSDSimpleTypeDefinition) {
+                    String dataType = xsdTypeDefinition.getQName();
+                    if (!XSDConstants.isSchemaForSchemaNamespace(xsdTypeDefinition.getTargetNamespace())
+                            && xsdTypeDefinition.getBaseType() != null) {
+                        if (!"xs:anySimpleType".equals(xsdTypeDefinition.getBaseType().getQName())) {
+                            dataType = xsdTypeDefinition.getBaseType().getQName();
+                        } else if (xsdTypeDefinition instanceof XSDSimpleTypeDefinition) {
+                            dataType = getVarietyType(((XSDSimpleTypeDefinition) xsdTypeDefinition).getVariety());
+                        }
+                    }
+                    node.setDataType(dataType);
                 }
                 List<String> namespaceList = new ArrayList(namespaceToPrefix.keySet());
                 Collections.reverse(namespaceList);
@@ -565,7 +591,11 @@ public class XSDPopulationUtil2 implements IXSDPopulationUtil {
             particleToTreeNode.clear();
         }
 
-        return rootNodes.get(0);
+        if (rootNodes.isEmpty()) {
+            return null;
+        } else {
+            return rootNodes.get(0);
+        }
     }
 
     private void addSubstitutionDetails(XSDSchema xsdSchema, ATreeNode parentNode, XSDElementDeclaration elementDeclaration,
@@ -760,11 +790,14 @@ public class XSDPopulationUtil2 implements IXSDPopulationUtil {
             ATreeNode childNode = new ATreeNode();
             childNode.setValue(attributeDeclarationName);
             childNode.setType(ATreeNode.ATTRIBUTE_TYPE);
-            String dataType = xsdAttributeDeclaration.getTypeDefinition().getQName();
-            XSDTypeDefinition baseType = xsdAttributeDeclaration.getTypeDefinition().getBaseType();
-            if (!XSDConstants.isSchemaForSchemaNamespace(xsdAttributeDeclaration.getTypeDefinition().getTargetNamespace())) {
+            XSDSimpleTypeDefinition typeDefinition = xsdAttributeDeclaration.getTypeDefinition();
+            String dataType = typeDefinition.getQName();
+            XSDTypeDefinition baseType = typeDefinition.getBaseType();
+            if (!XSDConstants.isSchemaForSchemaNamespace(typeDefinition.getTargetNamespace())) {
                 if (baseType != null && !"xs:anySimpleType".equals(baseType.getQName())) { //$NON-NLS-1$
                     dataType = baseType.getQName();
+                } else if (typeDefinition instanceof XSDSimpleTypeDefinition) {
+                    dataType = getVarietyType(((XSDSimpleTypeDefinition) typeDefinition).getVariety());
                 }
             }
             if (dataType != null && dataType.length() > 0) {

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/RepositorySystemFactory.java

@@ -35,6 +35,7 @@ import org.eclipse.aether.util.artifact.SubArtifact;
 import org.eclipse.aether.util.listener.ChainedRepositoryListener;
 import org.eclipse.aether.util.listener.ChainedTransferListener;
 import org.eclipse.aether.util.repository.AuthenticationBuilder;
+import org.talend.designer.maven.aether.util.TalendAetherProxySelector;
 
 /**
  * created by wchen on Aug 10, 2017 Detailled comment
@@ -72,6 +73,8 @@ public class RepositorySystemFactory {
                     .setLocalRepositoryManager(system.newLocalRepositoryManager(repositorySystemSession, localRepo));
             repositorySystemSession.setTransferListener(new ChainedTransferListener());
             repositorySystemSession.setRepositoryListener(new ChainedRepositoryListener());
+            repositorySystemSession.setProxySelector(new TalendAetherProxySelector());
+            sessions.put(localRepo, repositorySystemSession);
         }
 
         return repositorySystemSession;
@@ -107,6 +110,7 @@ public class RepositorySystemFactory {
         Authentication auth = new AuthenticationBuilder().addUsername(userName).addPassword(password).build();
         RemoteRepository distRepo = new RemoteRepository.Builder(repositoryId, "default", repositoryUrl).setAuthentication(auth)
                 .build();
+        distRepo = new RemoteRepository.Builder(distRepo).setProxy(new TalendAetherProxySelector().getProxy(distRepo)).build();
 
         deployRequest.setRepository(distRepo);
 

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/DynamicDistributionAetherUtils.java

@@ -152,6 +152,7 @@ public class DynamicDistributionAetherUtils {
             builder = builder.setAuthentication(auth);
         }
         RemoteRepository central = builder.build();
+        central = new RemoteRepository.Builder(central).setProxy(new TalendAetherProxySelector().getProxy(central)).build();
 
         CollectRequest collectRequest = new CollectRequest();
         collectRequest.setRoot(dependency);
@@ -307,6 +308,7 @@ public class DynamicDistributionAetherUtils {
             builder = builder.setAuthentication(auth);
         }
         RemoteRepository central = builder.build();
+        central = new RemoteRepository.Builder(central).setProxy(new TalendAetherProxySelector().getProxy(central)).build();
 
         VersionRangeRequest verRangeRequest = new VersionRangeRequest();
         verRangeRequest.addRepository(central);
@@ -353,6 +355,7 @@ public class DynamicDistributionAetherUtils {
             builder = builder.setAuthentication(auth);
         }
         RemoteRepository central = builder.build();
+        central = new RemoteRepository.Builder(central).setProxy(new TalendAetherProxySelector().getProxy(central)).build();
 
         VersionRangeRequest verRangeRequest = new VersionRangeRequest();
         verRangeRequest.addRepository(central);
@@ -426,6 +429,7 @@ public class DynamicDistributionAetherUtils {
 
         LocalRepository localRepo = new LocalRepository(repositoryPath);
         session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo));
+        session.setProxySelector(new TalendAetherProxySelector());
 
         updateDependencySelector(session, monitor);
 

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/MavenLibraryResolverProvider.java

@@ -14,6 +14,7 @@ package org.talend.designer.maven.aether.util;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Consumer;
 
 import org.apache.maven.model.License;
 import org.apache.maven.model.Model;
@@ -36,6 +37,7 @@ import org.eclipse.aether.transport.file.FileTransporterFactory;
 import org.eclipse.aether.transport.http.HttpTransporterFactory;
 import org.eclipse.aether.util.repository.AuthenticationBuilder;
 import org.eclipse.m2e.core.MavenPlugin;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.nexus.ArtifactRepositoryBean;
 import org.talend.core.nexus.TalendLibsServerManager;
 import org.talend.core.runtime.maven.MavenArtifact;
@@ -82,6 +84,8 @@ public class MavenLibraryResolverProvider {
             defaultRemoteRepository = new RemoteRepository.Builder("talend", "default", talendServer.getRepositoryURL()) //$NON-NLS-1$ //$NON-NLS-2$
                     .setAuthentication(authentication).build();
         }
+        defaultRemoteRepository = new RemoteRepository.Builder(defaultRemoteRepository)
+                .setProxy(new TalendAetherProxySelector().getProxy(defaultRemoteRepository)).build();
     }
 
     public ArtifactResult resolveArtifact(MavenArtifact aritfact) throws Exception {
@@ -121,16 +125,28 @@ public class MavenLibraryResolverProvider {
     }
 
     public RemoteRepository getRemoteRepositroy(MavenArtifact aritfact) {
+        RemoteRepository remoteRepository = null;
+        Consumer<RemoteRepository> update = null;
         if (aritfact != null && aritfact.getRepositoryUrl() != null) {
             if (urlToRepositoryMap.containsKey(aritfact.getRepositoryUrl())) {
-                return urlToRepositoryMap.get(aritfact.getRepositoryUrl());
+                remoteRepository = urlToRepositoryMap.get(aritfact.getRepositoryUrl());
+            } else {
+                remoteRepository = buildRemoteRepository(aritfact);
+                urlToRepositoryMap.put(aritfact.getRepositoryUrl(), remoteRepository);
             }
-
-            RemoteRepository repository = buildRemoteRepository(aritfact);
-            urlToRepositoryMap.put(aritfact.getRepositoryUrl(), repository);
-            return repository;
+            update = (r) -> urlToRepositoryMap.put(aritfact.getRepositoryUrl(), r);
+        } else {
+            remoteRepository = defaultRemoteRepository;
+            update = (r) -> defaultRemoteRepository = r;
         }
-        return defaultRemoteRepository;
+        try {
+            remoteRepository = new RemoteRepository.Builder(remoteRepository)
+                    .setProxy(defaultRepoSystemSession.getProxySelector().getProxy(remoteRepository)).build();
+            update.accept(remoteRepository);
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+        return remoteRepository;
     }
 
     private RemoteRepository buildRemoteRepository(MavenArtifact aritfact) {
@@ -143,6 +159,8 @@ public class MavenLibraryResolverProvider {
             repository = new RemoteRepository.Builder("talend", "default", aritfact.getRepositoryUrl()) //$NON-NLS-1$ //$NON-NLS-2$
                     .setAuthentication(authentication).build();
         }
+        repository = new RemoteRepository.Builder(repository).setProxy(new TalendAetherProxySelector().getProxy(repository))
+                .build();
         return repository;
     }
 
@@ -160,6 +178,7 @@ public class MavenLibraryResolverProvider {
 
         LocalRepository localRepo = new LocalRepository( /* "target/local-repo" */target);
         session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo));
+        session.setProxySelector(new TalendAetherProxySelector());
 
         return session;
     }

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/TalendAetherProxySelector.java

@@ -0,0 +1,98 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.designer.maven.aether.util;
+
+import org.eclipse.aether.repository.Authentication;
+import org.eclipse.aether.repository.Proxy;
+import org.eclipse.aether.repository.ProxySelector;
+import org.eclipse.aether.repository.RemoteRepository;
+import org.eclipse.aether.util.repository.AuthenticationBuilder;
+import org.eclipse.aether.util.repository.DefaultProxySelector;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.TalendProxySelector;
+
+
+/**
+ * DOC cmeng  class global comment. Detailled comment
+ */
+public class TalendAetherProxySelector implements ProxySelector {
+
+    private boolean isTalendDebug = false;
+
+    public TalendAetherProxySelector() {
+        isTalendDebug = Boolean.valueOf(System.getProperty(TalendProxySelector.PROP_PRINT_LOGS, "false"));
+    }
+
+    @Override
+    public Proxy getProxy(RemoteRepository repository) {
+        /**
+         * Update each time in case the settings are changed
+         */
+        Proxy proxy = createProxySelector().getProxy(repository);
+        if (isTalendDebug) {
+            try {
+                if (repository != null) {
+                    String proxyStr = "";
+                    if (proxy != null) {
+                        proxyStr = proxy.getType() + " " + proxy.toString() + ", proxy user: "
+                                + (proxy.getAuthentication() != null ? "..." : "<empty>");
+                    }
+                    ExceptionHandler.log("Aether proxy> host: " + repository.getHost() + ", proxy: " + proxyStr);
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+        }
+        return proxy;
+    }
+
+    private DefaultProxySelector createProxySelector() {
+        DefaultProxySelector proxySelector = new DefaultProxySelector();
+        javaDefaultProxy(proxySelector);
+        return proxySelector;
+    }
+
+    private void javaDefaultProxy(DefaultProxySelector proxySelector) {
+        String[] schemas = new String[] { "http", "https" };
+        for (String schema : schemas) {
+            Proxy proxy = createProxy(schema);
+            if (proxy != null) {
+                proxySelector.add(proxy, System.getProperty(schema + ".nonProxyHosts"));
+            }
+        }
+    }
+
+    private org.eclipse.aether.repository.Proxy createProxy(String schema) {
+        String proxyHost = System.getProperty(schema + ".proxyHost");
+        if (proxyHost == null) {
+            return null;
+        }
+        String proxyUser = System.getProperty(schema + ".proxyUser");
+        String proxyPassword = System.getProperty(schema + ".proxyPassword");
+        int proxyPort = Integer.parseInt(System.getProperty(schema + ".proxyPort", "8080"));
+
+        Authentication authentication = createAuthentication(proxyUser, proxyPassword);
+        org.eclipse.aether.repository.Proxy proxyObj = new org.eclipse.aether.repository.Proxy(schema, proxyHost, proxyPort,
+                authentication);
+        return proxyObj;
+    }
+
+    private Authentication createAuthentication(String proxyUser, String proxyPassword) {
+        Authentication authentication = null;
+        if (proxyUser != null) {
+            authentication = new AuthenticationBuilder().addUsername(proxyUser).addPassword(proxyPassword).build();
+        }
+        return authentication;
+    }
+
+}

main/plugins/org.talend.designer.maven.tos/resources/tacokit/pom.xml

@@ -10,7 +10,7 @@
 	<artifactId>studio-tacokit-dependencies</artifactId>
 	<packaging>pom</packaging>
 	<properties>
-		<tacokit.components.version>1.1.0</tacokit.components.version>
+		<tacokit.components.version>1.2.3</tacokit.components.version>
 	</properties>
 	<repositories>
 		<repository>

main/plugins/org.talend.designer.maven.tos/resources/tcompv1/pom.xml

@@ -11,7 +11,7 @@
     <packaging>pom</packaging>
     
     <properties>
-        <tcomp.version>1.1.10</tcomp.version>
+        <tcomp.version>1.1.14</tcomp.version>
         <slf4j.version>1.7.25</slf4j.version>
     </properties>
     
@@ -91,6 +91,24 @@
             <artifactId>slf4j-standard</artifactId>
             <version>${tcomp.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.talend.sdk.component</groupId>
+            <artifactId>component-runtime</artifactId>
+            <type>pom</type>
+            <version>${tcomp.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.talend.sdk.component</groupId>
+            <artifactId>component-studio</artifactId>
+            <type>pom</type>
+            <version>${tcomp.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.talend.sdk.component</groupId>
+            <artifactId>container</artifactId>
+            <type>pom</type>
+            <version>${tcomp.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-jdk14</artifactId>

main/plugins/org.talend.designer.maven.ui/src/main/java/messages.properties

@@ -11,4 +11,5 @@ ProjectPomProjectSettingPage_FilterPomLabel=Filter to use to generate poms:
 ProjectPomProjectSettingPage_FilterErrorMessage=Invalid filter: {0}
 ProjectPomProjectSettingPage.syncAllPomsButtonText=Force full re-synchronize poms
 AbstractPersistentProjectSettingPage.syncAllPoms=Do you want to update all poms? \n This operation might take long time depends on your project size.
-MavenProjectSettingPage.filterExampleMessage=Filter examples:\nlabel=myJob       \t\t\t\t=> Generate only the job named "myJob"\n!(label=myJob)    \t\t\t\t=> Generate any job except the one named "myJob"\n(path=folder1/folder2)  \t\t\t=> Generate any job in the folder "folder1/folder2"\n(path=folder1/folder2)or(label=myJob)\t=> Generate any job in the folder "folder1/folder2" or named "myJob"\n(label=myJob)and(version=0.2)	\t=> Generate only the job named "myJob" with version 0.2\n!((label=myJob)and(version=0.1))	\t=> Generate every jobs except the "myJob" version 0.1
+MavenProjectSettingPage.filterExampleMessage=Filter examples:\nlabel=myJob       \t\t\t\t=> Generate only the job named "myJob"\n!(label=myJob)    \t\t\t\t=> Generate any job except the one named "myJob"\n(path=folder1/folder2)  \t\t\t=> Generate any job in the folder "folder1/folder2"\n(path=folder1/folder2)or(label=myJob)\t=> Generate any job in the folder "folder1/folder2" or named "myJob"\n(label=myJob)and(version=0.2)	\t=> Generate only the job named "myJob" with version 0.2\n!((label=myJob)and(version=0.1))	\t=> Generate every jobs except the "myJob" version 0.1
+MavenProjectSettingPage.refModuleText=Set reference project modules in profile

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/MavenUIService.java

@@ -14,6 +14,7 @@ package org.talend.designer.maven.ui;
 
 import java.util.Dictionary;
 import java.util.Hashtable;
+import java.util.Set;
 
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IProgressMonitor;
@@ -118,6 +119,17 @@ public class MavenUIService implements IMavenUIService {
         String studioUserSettingsFile = MavenPlugin.getMavenConfiguration().getUserSettingsFile();
         // apply the user settings to MavenResolver
         Dictionary<String, String> props = new Hashtable<String, String>();
+        Set<Object> keySet = System.getProperties().keySet();
+        if (keySet != null) {
+            for (Object keyObj : keySet) {
+                if (keyObj instanceof String) {
+                    String key = keyObj.toString();
+                    if (key.startsWith("org.ops4j.pax.url.mvn.")) {
+                        props.put(key, System.getProperty(key));
+                    }
+                }
+            }
+        }
         if (studioUserSettingsFile != null && !"".equals(studioUserSettingsFile)) {
             // change back to use the user settings after Pax-url-eather fix the space bug
             props.put("org.ops4j.pax.url.mvn.settings", studioUserSettingsFile);

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/setting/preference/M2eUserSettingForTalendLoginTask.java

@@ -294,20 +294,25 @@ public class M2eUserSettingForTalendLoginTask extends AbstractLoginTask {
         boolean isLocal = isLocalRepository();
         IPath localRepoPath = null;
         if (!isLocal) {
-            String mvnHome = System.getenv("M2_HOME"); //$NON-NLS-1$
-            if (mvnHome == null) {
-                mvnHome = System.getenv("MAVEN_HOME"); //$NON-NLS-1$
-            }
-            if (StringUtils.isNotBlank(mvnHome)) {
-                File globalSettings = new File(mvnHome).toPath().resolve("conf").resolve("settings.xml").toFile(); //$NON-NLS-1$ //$NON-NLS-2$
-                if (globalSettings.exists()) {
-                    DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
-                    Document document = builder.parse(globalSettings);
-                    Node node = document.getElementsByTagName("localRepository").item(0); //$NON-NLS-1$
-                    if (node != null) {
-                        String repoPath = node.getTextContent();
-                        if (StringUtils.isNotBlank(repoPath)) {
-                            localRepoPath = new Path(repoPath);
+            String customMavenRepoistory = System.getProperty("maven.local.repository");
+            if (customMavenRepoistory != null) {
+                localRepoPath = new Path(customMavenRepoistory);
+            } else {
+                String mvnHome = System.getenv("M2_HOME"); //$NON-NLS-1$
+                if (mvnHome == null) {
+                    mvnHome = System.getenv("MAVEN_HOME"); //$NON-NLS-1$
+                }
+                if (StringUtils.isNotBlank(mvnHome)) {
+                    File globalSettings = new File(mvnHome).toPath().resolve("conf").resolve("settings.xml").toFile(); //$NON-NLS-1$ //$NON-NLS-2$
+                    if (globalSettings.exists()) {
+                        DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
+                        Document document = builder.parse(globalSettings);
+                        Node node = document.getElementsByTagName("localRepository").item(0); //$NON-NLS-1$
+                        if (node != null) {
+                            String repoPath = node.getTextContent();
+                            if (StringUtils.isNotBlank(repoPath)) {
+                                localRepoPath = new Path(repoPath);
+                            }
                         }
                     }
                 }

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/setting/project/initializer/MavenScriptsProjectSettingInitializer.java

@@ -32,6 +32,7 @@ import org.talend.designer.maven.template.AbstractMavenTemplateManager;
 import org.talend.designer.maven.template.MavenTemplateManager;
 import org.talend.designer.maven.tools.extension.PomExtensionRegistry;
 import org.talend.designer.maven.ui.DesignerMavenUiPlugin;
+import org.talend.designer.maven.utils.PomUtil;
 
 /**
  * DOC ggu class global comment. Detailled comment
@@ -68,6 +69,7 @@ public class MavenScriptsProjectSettingInitializer extends AbstractProjectPrefer
                 Model model = MavenPlugin.getMavenModelManager().readMavenModel(stream);
                 PomExtensionRegistry.getInstance().updatePomTemplate(model);
                 ByteArrayOutputStream out = new ByteArrayOutputStream();
+                PomUtil.sortModules(model);
                 MavenPlugin.getMaven().writeModel(model, out);
                 String content = out.toString(TalendMavenConstants.DEFAULT_ENCODING);
                 if (content != null) {

main/plugins/org.talend.designer.maven.ui/src/main/java/org/talend/designer/maven/ui/setting/project/page/MavenProjectSettingPage.java

@@ -50,6 +50,8 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
 
 	private IPreferenceStore preferenceStore;
 
+    private Button useProfileModuleCheckbox;
+
 	public MavenProjectSettingPage() {
 		noDefaultAndApplyButton();
 	}
@@ -84,6 +86,10 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
         filterExampleLable.setText(Messages.getString("MavenProjectSettingPage.filterExampleMessage")); //$NON-NLS-1$
         filterExampleLable.setLayoutData(new GridData(SWT.FILL, SWT.CENTER, true, false));
 
+        useProfileModuleCheckbox = new Button(parent, SWT.CHECK);
+        useProfileModuleCheckbox.setText(Messages.getString("MavenProjectSettingPage.refModuleText")); //$NON-NLS-1$
+        useProfileModuleCheckbox.setSelection(preferenceStore.getBoolean(MavenConstants.USE_PROFILE_MODULE));
+
         filterText.setText(filter);
 		filterText.addModifyListener(new ModifyListener() {
 
@@ -115,6 +121,7 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
 			public void widgetSelected(SelectionEvent event) {
 				try {
                     preferenceStore.setValue(MavenConstants.POM_FILTER, getRealVersionFilter(filter));
+                    preferenceStore.setValue(MavenConstants.USE_PROFILE_MODULE, useProfileModuleCheckbox.getSelection());
 					new AggregatorPomsHelper().syncAllPoms();
 				} catch (Exception e) {
 					ExceptionHandler.process(e);
@@ -133,6 +140,7 @@ public class MavenProjectSettingPage extends AbstractProjectSettingPage {
 		boolean ok = super.performOk();
 		if (preferenceStore != null) {
             preferenceStore.setValue(MavenConstants.POM_FILTER, getRealVersionFilter(filter));
+            preferenceStore.setValue(MavenConstants.USE_PROFILE_MODULE, useProfileModuleCheckbox.getSelection());
 		}
 		return ok;
 	}

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/launch/MavenCommandLauncher.java

@@ -13,6 +13,8 @@
 package org.talend.designer.maven.launch;
 
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.commons.lang.StringUtils;
 import org.eclipse.core.resources.IContainer;
@@ -95,6 +97,12 @@ public abstract class MavenCommandLauncher {
 
     private Map<String, Object> argumentsMap;
 
+    private static final String REGEX_TEST_CASE_FAILURES_STR = "\\[ERROR\\] Tests run:.*\\[ERROR\\] There are test\\p{Print}+\n";
+
+    private static final Pattern REGEX_TEST_CASE_FAILURES = Pattern.compile(REGEX_TEST_CASE_FAILURES_STR, Pattern.DOTALL);
+
+    private boolean ignoreTestFailure = false;
+
     public MavenCommandLauncher(String goals) {
         super();
         Assert.isNotNull(goals);
@@ -103,6 +111,14 @@ public abstract class MavenCommandLauncher {
         this.debugOutput = MavenPlugin.getMavenConfiguration().isDebugOutput();
     }
 
+    public boolean isIgnoreTestFailure() {
+        return ignoreTestFailure;
+    }
+
+    public void setIgnoreTestFailure(boolean ignoreTestFailure) {
+        this.ignoreTestFailure = ignoreTestFailure;
+    }
+
     protected String getGoals() {
         return goals;
     }
@@ -205,6 +221,12 @@ public abstract class MavenCommandLauncher {
                 workingCopy.setAttribute(IJavaLaunchConfigurationConstants.ATTR_VM_ARGUMENTS, vmargs);
             }
 
+            // ignore test failures
+            if (this.ignoreTestFailure) {
+                workingCopy.setAttribute(IJavaLaunchConfigurationConstants.ATTR_VM_ARGUMENTS, "-Dmaven.test.failure.ignore=true "
+                        + workingCopy.getAttribute(IJavaLaunchConfigurationConstants.ATTR_VM_ARGUMENTS, ""));
+            }
+
             String programArgs = getArgumentValue(TalendProcessArgumentConstant.ARG_PROGRAM_ARGUMENTS);
             if (StringUtils.isNotEmpty(programArgs)) {
                 workingCopy.setAttribute(IJavaLaunchConfigurationConstants.ATTR_PROGRAM_ARGUMENTS, programArgs);
@@ -318,7 +340,18 @@ public abstract class MavenCommandLauncher {
 
         if (TalendMavenConstants.GOAL_INSTALL.equals(launchConfiguration.getAttribute(MavenLaunchConstants.ATTR_GOALS, ""))) {
             if (errors.length() != 0) {
-                throw new Exception(errors.toString());
+                String remainingErr = errors.toString();
+                if (this.ignoreTestFailure) {
+                    Matcher m = REGEX_TEST_CASE_FAILURES.matcher(errors);
+                    int matchIdx = 0;
+                    while (m.find()) {
+                        matchIdx = m.end();
+                    }
+                    remainingErr = errors.substring(matchIdx);
+                }
+                if (remainingErr.trim().length() > 0) {
+                    throw new Exception(remainingErr.toString());
+                }
             }
         }
     }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/AggregatorPomsHelper.java

@@ -12,24 +12,23 @@
 // ============================================================================
 package org.talend.designer.maven.tools;
 
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_AGGREGATORS;
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_BEANS;
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_CODES;
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_JOBS;
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_PIGUDFS;
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_POMS;
-import static org.talend.designer.maven.model.TalendJavaProjectConstants.DIR_ROUTINES;
+import static org.talend.designer.maven.model.TalendJavaProjectConstants.*;
 
 import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
-import java.util.ListIterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.maven.model.Activation;
 import org.apache.maven.model.Model;
+import org.apache.maven.model.Profile;
 import org.eclipse.core.resources.IContainer;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
@@ -56,7 +55,6 @@ import org.talend.core.context.Context;
 import org.talend.core.context.RepositoryContext;
 import org.talend.core.model.general.ILibrariesService;
 import org.talend.core.model.general.ModuleNeeded;
-import org.talend.core.model.general.Project;
 import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.ProjectReference;
@@ -77,6 +75,7 @@ import org.talend.core.runtime.services.IFilterService;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.designer.core.ICamelDesignerCoreService;
 import org.talend.designer.maven.launch.MavenPomCommandLauncher;
+import org.talend.designer.maven.model.MavenSystemFolders;
 import org.talend.designer.maven.model.TalendJavaProjectConstants;
 import org.talend.designer.maven.model.TalendMavenConstants;
 import org.talend.designer.maven.template.MavenTemplateManager;
@@ -110,27 +109,25 @@ public class AggregatorPomsHelper {
         return projectTechName;
     }
 
-    public void createRootPom(List<String> modules, boolean force, IProgressMonitor monitor) throws Exception {
+    public void createRootPom(Model model, boolean force, IProgressMonitor monitor)
+            throws Exception {
         IFile pomFile = getProjectRootPom();
         if (force || !pomFile.exists()) {
-            Map<String, Object> parameters = new HashMap<String, Object>();
-            parameters.put(MavenTemplateManager.KEY_PROJECT_NAME, projectTechName);
-            Model model = MavenTemplateManager.getCodeProjectTemplateModel(parameters);
-            if (modules != null && !modules.isEmpty()) {
-                model.setModules(modules);
-            }
             PomUtil.savePom(monitor, model, pomFile);
         }
     }
 
     public void createRootPom(IProgressMonitor monitor) throws Exception {
+        Model newModel = getCodeProjectTemplateModel();
         IFile pomFile = getProjectRootPom();
-        List<String> modules = null;
         if (pomFile != null && pomFile.exists()) {
-            Model model = MavenPlugin.getMavenModelManager().readMavenModel(pomFile);
-            modules = model.getModules();
+            Model oldModel = MavenPlugin.getMavenModelManager().readMavenModel(pomFile);
+            List<Profile> profiles = oldModel.getProfiles().stream()
+                    .filter(profile -> matchModuleProfile(profile.getId(), projectTechName)).collect(Collectors.toList());
+            newModel.setModules(oldModel.getModules());
+            newModel.getProfiles().addAll(profiles);
         }
-        createRootPom(modules, true, monitor);
+        createRootPom(newModel, true, monitor);
     }
 
     public void installRootPom(boolean force) throws Exception {
@@ -290,37 +287,40 @@ public class AggregatorPomsHelper {
         }
     }
 
-    public void updateRefProjectModules(List<ProjectReference> references) {
+    public void updateRefProjectModules(List<ProjectReference> references, IProgressMonitor monitor) {
         if (!needUpdateRefProjectModules()) {
             return;
         }
         try {
-            List<String> modules = new ArrayList<>();
-            for (ProjectReference reference : references) {
-                String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
-                String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
-                modules.add(modulePath);
-            }
-
-            Project mainProject = ProjectManager.getInstance().getCurrentProject();
-            IFolder mainPomsFolder = new AggregatorPomsHelper(mainProject.getTechnicalLabel()).getProjectPomsFolder();
-            IFile mainPomFile = mainPomsFolder.getFile(TalendMavenConstants.POM_FILE_NAME);
-
-            Model model = MavenPlugin.getMavenModelManager().readMavenModel(mainPomFile);
-            List<String> oldModules = model.getModules();
-            if (oldModules == null) {
-                oldModules = new ArrayList<>();
-            }
-            ListIterator<String> iterator = oldModules.listIterator();
-            while (iterator.hasNext()) {
-                String modulePath = iterator.next();
-                if (modulePath.startsWith("../../")) { //$NON-NLS-1$
-                    iterator.remove();
+            Model model = MavenPlugin.getMavenModelManager().readMavenModel(getProjectRootPom());
+            if (PomIdsHelper.useProfileModule()) {
+                List<Profile> profiles = collectRefProjectProfiles(references);
+                Iterator<Profile> iterator = model.getProfiles().listIterator();
+                while (iterator.hasNext()) {
+                    Profile profile = iterator.next();
+                    if (matchModuleProfile(profile.getId(), projectTechName)) {
+                        iterator.remove();
+                    }
                 }
+                model.getProfiles().addAll(profiles);
+            } else {
+                List<String> refPrjectModules = new ArrayList<>();
+                references.forEach(reference -> {
+                    String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
+                    String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
+                    refPrjectModules.add(modulePath);
+                });
+                List<String> modules = model.getModules();
+                Iterator<String> iterator = modules.listIterator();
+                while (iterator.hasNext()) {
+                    String modulePath = iterator.next();
+                    if (modulePath.startsWith("../../")) { //$NON-NLS-1$
+                        iterator.remove();
+                    }
+                }
+                modules.addAll(refPrjectModules);
             }
-            oldModules.addAll(modules);
-
-            PomUtil.savePom(null, model, mainPomFile);
+            createRootPom(model, true, monitor);
         } catch (Exception e) {
             ExceptionHandler.process(e);
         }
@@ -494,6 +494,10 @@ public class AggregatorPomsHelper {
         return null;
     }
 
+    public IFolder getCodeSrcFolder(ERepositoryObjectType codeType) {
+        return getCodeFolder(codeType).getFolder(MavenSystemFolders.JAVA.getPath());
+    }
+
     public IFolder getProcessFolder(ERepositoryObjectType type) {
         return getProjectPomsFolder().getFolder(DIR_JOBS).getFolder(type.getFolder());
     }
@@ -684,6 +688,23 @@ public class AggregatorPomsHelper {
         new ProgressMonitorDialog(Display.getDefault().getActiveShell()).run(true, true, runnableWithProgress);
     }
 
+    public void syncParentJobPomsForPropertyChange(Property property) {
+        IRunProcessService runProcessService = getRunProcessService();
+        ProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
+        List<Relation> itemsHaveRelationWith = RelationshipItemBuilder.getInstance().getItemsHaveRelationWith(property.getId(),
+                property.getVersion());
+        try {
+            for (Relation relation : itemsHaveRelationWith) {
+                IRepositoryViewObject object = factory.getSpecificVersion(relation.getId(), relation.getVersion(), true);
+                if (runProcessService != null) {
+                    runProcessService.generatePom(object.getProperty().getItem());
+                }
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
     private String getModulePath(IFile pomFile) {
         IFile parentPom = getProjectRootPom();
         if (parentPom != null) {
@@ -693,47 +714,48 @@ public class AggregatorPomsHelper {
         return null;
     }
 
-    private void collectModules(List<String> modules) {
-        IRunProcessService service = getRunProcessService();
-        if (service != null) {
-            modules.add(
-                    getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.ROUTINES).getProjectPom()));
-            if (ProcessUtils.isRequiredPigUDFs(null)) {
-                modules.add(
-                        getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.PIG_UDF).getProjectPom()));
-            }
-            if (ProcessUtils.isRequiredBeans(null)) {
-                modules.add(getModulePath(service
-                        .getTalendCodeJavaProject(ERepositoryObjectType.valueOf("BEANS")) //$NON-NLS-1$
-                        .getProjectPom()));
-            }
+    private List<Profile> collectRefProjectProfiles(List<ProjectReference> references) throws CoreException {
+        if (!needUpdateRefProjectModules()) {
+            Model model = MavenPlugin.getMavenModelManager().readMavenModel(getProjectRootPom());
+            List<Profile> profiles = model.getProfiles();
+            return profiles.stream().filter(profile -> matchModuleProfile(profile.getId(), projectTechName))
+                    .collect(Collectors.toList());
         }
-        if (needUpdateRefProjectModules()) {
-            List<ProjectReference> references =
-                    ProjectManager.getInstance().getCurrentProject().getProjectReferenceList(true);
-            for (ProjectReference reference : references) {
-                String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
-                String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
-                modules.add(modulePath);
-            }
-        } else {
-            Project mainProject = ProjectManager.getInstance().getCurrentProject();
-            IFolder mainPomsFolder = new AggregatorPomsHelper(mainProject.getTechnicalLabel()).getProjectPomsFolder();
-            IFile mainPomFile = mainPomsFolder.getFile(TalendMavenConstants.POM_FILE_NAME);
-            try {
-                Model model = MavenPlugin.getMavenModelManager().readMavenModel(mainPomFile);
-                List<String> oldModules = model.getModules();
-                if (oldModules != null) {
-                    for (String modulePath : oldModules) {
-                        if (modulePath.startsWith("../../")) { //$NON-NLS-1$
-                            modules.add(modulePath);
-                        }
-                    }
-                }
-            } catch (CoreException e) {
-                ExceptionHandler.process(e);
-            }
+        if (references == null) {
+            references = ProjectManager.getInstance().getCurrentProject().getProjectReferenceList(true);
         }
+        List<Profile> profiles = new ArrayList<>();
+        references.forEach(reference -> {
+            String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
+            String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
+            Profile profile = new Profile();
+            profile.setId((projectTechName + "_" + refProjectTechName).toLowerCase()); //$NON-NLS-1$
+            Activation activation = new Activation();
+            activation.setActiveByDefault(true);
+            profile.setActivation(activation);
+            profile.getModules().add(modulePath);
+            profiles.add(profile);
+        });
+        return profiles;
+    }
+
+    private List<String> collectRefProjectModules(List<ProjectReference> references) throws CoreException {
+        if (!needUpdateRefProjectModules()) {
+            Model model = MavenPlugin.getMavenModelManager().readMavenModel(getProjectRootPom());
+            return model.getModules().stream().filter(modulePath -> modulePath.startsWith("../../")) //$NON-NLS-1$
+                    .collect(Collectors.toList());
+        }
+        if (references == null) {
+            references = ProjectManager.getInstance().getCurrentProject().getProjectReferenceList(true);
+        }
+        List<String> modules = new ArrayList<>();
+        references.forEach(reference -> {
+            String refProjectTechName = reference.getReferencedProject().getTechnicalLabel();
+            String modulePath = "../../" + refProjectTechName + "/" + TalendJavaProjectConstants.DIR_POMS; //$NON-NLS-1$ //$NON-NLS-2$
+            modules.add(modulePath);
+        });
+        return modules;
+
     }
 
     public boolean needUpdateRefProjectModules() {
@@ -771,7 +793,13 @@ public class AggregatorPomsHelper {
         monitor.beginTask("", size); //$NON-NLS-1$
         // project pom
         monitor.subTask("Synchronize project pom"); //$NON-NLS-1$
-        createRootPom(null, true, monitor);
+        Model model = getCodeProjectTemplateModel();
+        if (PomIdsHelper.useProfileModule()) {
+            model.getProfiles().addAll(collectRefProjectProfiles(null));
+        } else {
+            model.getModules().addAll(collectRefProjectModules(null));
+        }
+        createRootPom(model, true, monitor);
         installRootPom(true);
         monitor.worked(1);
         if (monitor.isCanceled()) {
@@ -824,8 +852,9 @@ public class AggregatorPomsHelper {
         }
         // sync project pom again with all modules.
         monitor.subTask("Synchronize project pom with modules"); //$NON-NLS-1$
-        collectModules(modules);
-        createRootPom(modules, true, monitor);
+        collectCodeModules(modules);
+        model.getModules().addAll(modules);
+        createRootPom(model, true, monitor);
         installRootPom(true);
         monitor.worked(1);
         if (monitor.isCanceled()) {
@@ -834,6 +863,21 @@ public class AggregatorPomsHelper {
         monitor.done();
     }
 
+    private void collectCodeModules(List<String> modules) {
+        // collect codes modules
+        IRunProcessService service = getRunProcessService();
+        if (service != null) {
+            modules.add(getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.ROUTINES).getProjectPom()));
+            if (ProcessUtils.isRequiredPigUDFs(null)) {
+                modules.add(getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.PIG_UDF).getProjectPom()));
+            }
+            if (ProcessUtils.isRequiredBeans(null)) {
+                modules.add(getModulePath(service.getTalendCodeJavaProject(ERepositoryObjectType.valueOf("BEANS")) //$NON-NLS-1$
+                        .getProjectPom()));
+            }
+        }
+    }
+
     /**
      * Check if is a esb data service job
      *
@@ -862,6 +906,18 @@ public class AggregatorPomsHelper {
         return null;
     }
 
+    private Model getCodeProjectTemplateModel() {
+        Map<String, Object> parameters = new HashMap<String, Object>();
+        parameters.put(MavenTemplateManager.KEY_PROJECT_NAME, projectTechName);
+        return MavenTemplateManager.getCodeProjectTemplateModel(parameters);
+    }
+
+    public static boolean matchModuleProfile(String profileId, String projectTechName) {
+        // FIXME get profile id from extension point.
+        List<String> otherProfiles = Arrays.asList("docker", "cloud-publisher", "nexus"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+        return !otherProfiles.contains(profileId) && StringUtils.startsWithIgnoreCase(profileId, projectTechName + "_");
+    }
+
     private static IRunProcessService getRunProcessService() {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
             IRunProcessService runProcessService =

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/BuildCacheManager.java

@@ -223,8 +223,13 @@ public class BuildCacheManager {
 
                 String goal = (String) argumentsMap.get(TalendProcessArgumentConstant.ARG_GOAL);
                 MavenPomCommandLauncher mavenLauncher = new MavenPomCommandLauncher(pomFile, goal);
-                mavenLauncher.setSkipTests(true);
                 mavenLauncher.setArgumentsMap(argumentsMap);
+                if (isBuildJob()) {
+                    mavenLauncher.setIgnoreTestFailure(true);
+                } else {
+                    // run job, still skip tests anyway
+                    mavenLauncher.setSkipTests(true);
+                }
                 mavenLauncher.execute(monitor);
             } finally {
                 deleteBuildAggregatorPom();
@@ -375,4 +380,13 @@ public class BuildCacheManager {
         return aggregatorPomsHelper;
     }
 
+    private boolean isBuildJob() {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
+            IRunProcessService service = (IRunProcessService) GlobalServiceRegister.getDefault()
+                    .getService(IRunProcessService.class);
+            return service.isExportConfig();
+        }
+        return false;
+    }
+
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/ProcessorDependenciesManager.java

@@ -13,7 +13,7 @@
 package org.talend.designer.maven.tools;
 
 import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -24,16 +24,19 @@ import org.apache.maven.model.Dependency;
 import org.apache.maven.model.Exclusion;
 import org.apache.maven.model.Model;
 import org.eclipse.core.runtime.IProgressMonitor;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.ILibraryManagerService;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.process.IProcess2;
 import org.talend.core.model.properties.ProcessItem;
+import org.talend.core.model.properties.Property;
 import org.talend.core.runtime.process.LastGenerationInfo;
 import org.talend.core.runtime.process.TalendProcessOptionConstants;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.designer.maven.utils.PomUtil;
+import org.talend.designer.maven.utils.SortableDependency;
 import org.talend.designer.runprocess.IProcessor;
 import org.talend.designer.runprocess.ProcessorException;
 
@@ -45,44 +48,40 @@ public class ProcessorDependenciesManager {
 
     private final IProcessor processor;
 
+    private final Property property;
+
     public ProcessorDependenciesManager(IProcessor processor) {
         this.processor = processor;
+        property = processor.getProperty();
     }
 
     /**
      * Will add the dependencies to the maven model.
      */
+    @SuppressWarnings({ "rawtypes", "unchecked" })
     public boolean updateDependencies(IProgressMonitor progressMonitor, Model model) throws ProcessorException {
         try {
-            List neededDependencies = new ArrayList<Dependency>();
-            Set<ModuleNeeded> neededLibraries = getAllModuleNeededWithTestCase();
-            Set<String> uniquDependenciesSet = new HashSet<>();
-
-            for (ModuleNeeded module : neededLibraries) {
-                Dependency dependency = null;
-                // if (module.getDeployStatus() == ELibraryInstallStatus.DEPLOYED) {
-                // }
-                final String mavenUri = module.getMavenUri();
-                if (uniquDependenciesSet.contains(mavenUri)) {
-                    continue; // must be same GAV, avoid the different other attrs for modules
-                }
-                uniquDependenciesSet.add(mavenUri);
-                dependency = PomUtil.createModuleDependency(mavenUri);
-                if (dependency != null) {
-                    if (module.isExcludeDependencies()) {
-                        Exclusion exclusion = new Exclusion();
-                        exclusion.setGroupId("*"); //$NON-NLS-1$
-                        exclusion.setArtifactId("*"); //$NON-NLS-1$
-                        dependency.addExclusion(exclusion);
-                    }
-                    neededDependencies.add(dependency);
+            Set<ModuleNeeded> neededLibraries = new HashSet<>();
+            Set<String> uniqueDependencies = new HashSet<>();
+            Set<ModuleNeeded> jobModules = getProcessNeededModules();
+            Set<ModuleNeeded> testcaseModules = getTestcaseNeededModules(property);
+            neededLibraries.addAll(jobModules);
+            neededLibraries.addAll(testcaseModules);
+            if (!neededLibraries.isEmpty()) {
+                if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibraryManagerService.class)) {
+                    ILibraryManagerService repositoryBundleService = GlobalServiceRegister.getDefault()
+                            .getService(ILibraryManagerService.class);
+                    repositoryBundleService.installModules(neededLibraries, null);
                 }
             }
+            List neededDependencies = new ArrayList<>();
 
-            java.util.Collections.sort(neededDependencies);
+            neededDependencies.addAll(convertToDependency(jobModules, uniqueDependencies, false));
+            neededDependencies.addAll(convertToDependency(testcaseModules, uniqueDependencies, true));
+
+            Collections.sort(neededDependencies);
             boolean fresh = false;
-            if (processor.getProperty() != null && processor.getProperty().getItem() != null
-                    && processor.getProcess() instanceof IProcess2) {
+            if (property != null && property.getItem() != null && processor.getProcess() instanceof IProcess2) {
                 // is standard job.
                 fresh = true;
             }
@@ -93,6 +92,30 @@ public class ProcessorDependenciesManager {
         }
     }
 
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    private List convertToDependency(Set<ModuleNeeded> neededLibraries, Set<String> uniqueDependencies, boolean optional) {
+        List neededDependencies = new ArrayList<>();
+        for (ModuleNeeded module : neededLibraries) {
+            final String mavenUri = module.getMavenUri();
+            if (uniqueDependencies.contains(mavenUri)) {
+                continue; // must be same GAV, avoid the different other attrs for modules
+            }
+            uniqueDependencies.add(mavenUri);
+            Dependency dependency = PomUtil.createModuleDependency(mavenUri);
+            if (dependency != null) {
+                ((SortableDependency) dependency).setAssemblyOptional(optional);
+                if (module.isExcludeDependencies()) {
+                    Exclusion exclusion = new Exclusion();
+                    exclusion.setGroupId("*"); //$NON-NLS-1$
+                    exclusion.setArtifactId("*"); //$NON-NLS-1$
+                    dependency.addExclusion(exclusion);
+                }
+                neededDependencies.add(dependency);
+            }
+        }
+        return neededDependencies;
+    }
+
     /**
      *
      * DOC ggu Comment method "updateDependencies". add the job Needed Libraries for current model.
@@ -159,35 +182,35 @@ public class ProcessorDependenciesManager {
         return changed;
     }
 
-    private Set<ModuleNeeded> getAllModuleNeededWithTestCase() throws PersistenceException {
-        // add the job modules.
-        Set<ModuleNeeded> neededLibraries = new HashSet<>();
-        Collection<ModuleNeeded> modulesNeeded = LastGenerationInfo.getInstance()
+    private Set<ModuleNeeded> getProcessNeededModules() {
+        Set<ModuleNeeded> neededLibraries = LastGenerationInfo.getInstance()
                 .getModulesNeededPerJob(processor.getProcess().getId(), processor.getProcess().getVersion());
-        if (modulesNeeded.isEmpty()) {
-            modulesNeeded = processor.getNeededModules(TalendProcessOptionConstants.MODULES_WITH_JOBLET);
-        }
-        neededLibraries.addAll(modulesNeeded);
-
-        // add testcase modules
-        if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
-            ITestContainerProviderService testcontainerService = (ITestContainerProviderService) GlobalServiceRegister
-                    .getDefault().getService(ITestContainerProviderService.class);
-            Set<ModuleNeeded> testcaseModules = null;
-            if (processor.getProperty() != null && processor.getProperty().getItem() instanceof ProcessItem) {
-                ProcessItem item = (ProcessItem) processor.getProperty().getItem();
-                testcaseModules = testcontainerService.getAllJobTestcaseModules(item);
-                neededLibraries.addAll(testcaseModules);
-                if (testcaseModules != null) {
-                    if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibraryManagerService.class)) {
-                        ILibraryManagerService repositoryBundleService = (ILibraryManagerService) GlobalServiceRegister
-                                .getDefault().getService(ILibraryManagerService.class);
-                        repositoryBundleService.installModules(neededLibraries, null);
-                    }
-                }
-            }
+        if (neededLibraries.isEmpty()) {
+            neededLibraries = processor.getNeededModules(TalendProcessOptionConstants.MODULES_DEFAULT);
         }
         return neededLibraries;
     }
 
+    public static Set<ModuleNeeded> getTestcaseNeededModules(Property property) {
+        if (property == null || !(property.getItem() instanceof ProcessItem)) {
+            return Collections.emptySet();
+        }
+        Set<ModuleNeeded> testcaseModules = LastGenerationInfo.getInstance().getTestcaseModuleNeeded(property.getId(),
+                property.getVersion());
+        if (testcaseModules.isEmpty()) {
+            if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
+                ITestContainerProviderService testcontainerService = GlobalServiceRegister.getDefault()
+                        .getService(ITestContainerProviderService.class);
+                try {
+                    testcaseModules = testcontainerService.getAllJobTestcaseModules((ProcessItem) property.getItem());
+                    LastGenerationInfo.getInstance().setTestcaseModuleNeeded(property.getId(), property.getVersion(),
+                            testcaseModules);
+                } catch (PersistenceException e) {
+                    ExceptionHandler.process(e);
+                }
+            }
+        }
+        return testcaseModules;
+    }
+
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/ProjectPomManager.java

@@ -69,13 +69,7 @@ public class ProjectPomManager {
         if (monitor == null) {
             monitor = new NullProgressMonitor();
         }
-        Model projectModel = MODEL_MANAGER.readMavenModel(projectPomFile);
-        Model templateModel = MavenTemplateManager.getCodeProjectTemplateModel();
-        for (String module : projectModel.getModules()) {
-            templateModel.addModule(module);
-        }
-
-        PomUtil.savePom(monitor, templateModel, projectPomFile);
+        new AggregatorPomsHelper().createRootPom(monitor);
     }
 
     /**

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/AbstractMavenProcessorPom.java

@@ -45,6 +45,7 @@ import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenConstants;
 import org.talend.core.runtime.maven.MavenUrlHelper;
+import org.talend.core.runtime.process.TalendProcessArgumentConstant;
 import org.talend.core.runtime.projectsetting.IProjectSettingTemplateConstants;
 import org.talend.core.runtime.repository.build.IMavenPomCreator;
 import org.talend.core.ui.ITestContainerProviderService;
@@ -219,6 +220,29 @@ public abstract class AbstractMavenProcessorPom extends CreateMavenBundleTemplat
                     MavenArtifact mvnArtifact = MavenUrlHelper.parseMvnUrl(module.getMavenUri());
                     include.setValue(mvnArtifact.getGroupId() + ":" + mvnArtifact.getArtifactId()); //$NON-NLS-1$
                 }
+                
+                //removing digital signatures from uber jar
+                Xpp3Dom filters = new Xpp3Dom("filters"); //$NON-NLS-1$
+                Xpp3Dom filter = new Xpp3Dom("filter"); //$NON-NLS-1$
+                Xpp3Dom artifact = new Xpp3Dom("artifact"); //$NON-NLS-1$
+                artifact.setValue("*:*");
+                Xpp3Dom filterExcludes = new Xpp3Dom("excludes"); //$NON-NLS-1$
+                Xpp3Dom excludeSF = new Xpp3Dom("exclude");
+                excludeSF.setValue("META-INF/*.SF");
+                Xpp3Dom excludeDSA = new Xpp3Dom("exclude");
+                excludeDSA.setValue("META-INF/*.DSA");
+                Xpp3Dom excludeRSA = new Xpp3Dom("exclude");
+                excludeRSA.setValue("META-INF/*.RSA");
+
+                filterExcludes.addChild(excludeSF);
+                filterExcludes.addChild(excludeDSA);
+                filterExcludes.addChild(excludeRSA);
+                
+                filter.addChild(artifact);
+                filter.addChild(filterExcludes);
+                filters.addChild(filter);
+                configuration.addChild(filters);
+                
                 plugins.add(shade);
             }
         }
@@ -269,6 +293,7 @@ public abstract class AbstractMavenProcessorPom extends CreateMavenBundleTemplat
 
     protected void addChildrenDependencies(final List<Dependency> dependencies) {
         String parentId = getJobProcessor().getProperty().getId();
+
         final Set<JobInfo> clonedChildrenJobInfors = getJobProcessor().getBuildFirstChildrenJobs();
         for (JobInfo jobInfo : clonedChildrenJobInfors) {
             if (jobInfo.getFatherJobInfo() != null && jobInfo.getFatherJobInfo().getJobId().equals(parentId)) {
@@ -284,6 +309,10 @@ public abstract class AbstractMavenProcessorPom extends CreateMavenBundleTemplat
                     property = jobInfo.getProcessItem().getProperty();
                     groupId = PomIdsHelper.getJobGroupId(property);
                     artifactId = PomIdsHelper.getJobArtifactId(jobInfo);
+                    if ("OSGI".equals(property.getAdditionalProperties().get(TalendProcessArgumentConstant.ARG_BUILD_TYPE))) {
+                        artifactId = artifactId + "-bundle";
+                    }
+
                     version = PomIdsHelper.getJobVersion(property);
                     // try to get the pom version of children job and load from the pom file.
                     String childPomFileName = PomUtil.getPomFileName(jobInfo.getJobName(), jobInfo.getJobVersion());

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenJobPom.java

@@ -18,22 +18,30 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.talend.core.model.repository.ERepositoryObjectType;
 import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.apache.maven.artifact.versioning.ComparableVersion;
 import org.apache.maven.model.Activation;
 import org.apache.maven.model.Dependency;
 import org.apache.maven.model.Model;
+import org.apache.maven.model.Plugin;
+import org.apache.maven.model.PluginExecution;
 import org.apache.maven.model.Profile;
+import org.codehaus.plexus.util.xml.Xpp3Dom;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.runtime.CoreException;
@@ -56,9 +64,7 @@ import org.talend.core.model.properties.Project;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.repository.utils.ItemResourceUtil;
-import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenConstants;
-import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.core.runtime.process.ITalendProcessJavaProject;
 import org.talend.core.runtime.process.JobInfoProperties;
 import org.talend.core.runtime.process.LastGenerationInfo;
@@ -72,6 +78,7 @@ import org.talend.core.utils.TemplateFileUtils;
 import org.talend.designer.maven.model.TalendMavenConstants;
 import org.talend.designer.maven.template.ETalendMavenVariables;
 import org.talend.designer.maven.template.MavenTemplateManager;
+import org.talend.designer.maven.tools.ProcessorDependenciesManager;
 import org.talend.designer.maven.utils.PomIdsHelper;
 import org.talend.designer.maven.utils.PomUtil;
 import org.talend.designer.maven.utils.SortableDependency;
@@ -620,139 +627,103 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
     }
 
     protected void updateDependencySet(IFile assemblyFile) {
-        Map<String, Dependency> jobCoordinateMap = new HashMap<String, Dependency>();
-        Set<JobInfo> childrenJobInfo = new HashSet<>();
-        if (!hasLoopDependency()) {
-            childrenJobInfo = getJobProcessor().getBuildChildrenJobs();
-        }
-
-        // add children jobs
-        for (JobInfo jobInfo : childrenJobInfo) {
-            Property property = jobInfo.getProcessItem().getProperty();
-            String coordinate =
-                    getCoordinate(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo),
-                            MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(property));
-            Dependency dependency = getDependencyObject(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo), PomIdsHelper.getJobVersion(property),
-                            MavenConstants.PACKAGING_JAR, null);
-            jobCoordinateMap.put(coordinate, dependency);
-        }
-
-        // add parent job
-        Property parentProperty = this.getJobProcessor().getProperty();
-        String parentCoordinate =
-                getCoordinate(PomIdsHelper.getJobGroupId(parentProperty), PomIdsHelper.getJobArtifactId(parentProperty),
-                        MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(parentProperty));
-        Dependency parentDependency = getDependencyObject(PomIdsHelper.getJobGroupId(parentProperty), PomIdsHelper.getJobArtifactId(parentProperty), PomIdsHelper.getJobVersion(parentProperty),
-                        MavenConstants.PACKAGING_JAR, null);
-        jobCoordinateMap.put(parentCoordinate, parentDependency);
-
-        // add talend libraries and codes
-        Map<String, Dependency> talendLibCoordinateMap = new HashMap<String, Dependency>();
-        String projectTechName = ProjectManager.getInstance().getProject(parentProperty).getTechnicalLabel();
-        String projectGroupId = PomIdsHelper.getProjectGroupId(projectTechName);
-
-        // codes
-        List<Dependency> dependencies = new ArrayList<>();
-        addCodesDependencies(dependencies);
-        for (Dependency dependency : dependencies) {
-            talendLibCoordinateMap.put(getCoordinate(dependency), dependency);
-        }
-
-        // libraries
-        dependencies.clear();
-        Set<ModuleNeeded> modules = getJobProcessor()
-                .getNeededModules(TalendProcessOptionConstants.MODULES_WITH_JOBLET
-                        | TalendProcessOptionConstants.MODULES_EXCLUDE_SHADED);
-        for (ModuleNeeded module : modules) {
-            String mavenUri = module.getMavenUri();
-            Dependency dependency = PomUtil.createModuleDependency(mavenUri);
-            dependencies.add(dependency);
-        }
-        for (Dependency dependency : dependencies) {
-            if (MavenConstants.PACKAGING_POM.equals(dependency.getType())) {
-                continue;
-            }
-            String dependencyGroupId = dependency.getGroupId();
-            String coordinate = getCoordinate(dependency);
-            if (!jobCoordinateMap.containsKey(coordinate)) {
-                if (MavenConstants.DEFAULT_LIB_GROUP_ID.equals(dependencyGroupId)) {
-                    talendLibCoordinateMap.put(coordinate, dependency);
-                }
-            }
-        }
-
-        // add 3rd party libraries
-        Map<String, Dependency> _3rdDepLibMap = new HashMap<String, Dependency>();
+        Set<String> jobCoordinate = new HashSet<>();
+        Set<String> talendLibCoordinate = new HashSet<>();
+        Set<String> _3rdLibCoordinate = new HashSet<>();
         Map<String, Set<Dependency>> duplicateLibs = new HashMap<>();
-        for (Dependency dependency : dependencies) {
-            if (MavenConstants.PACKAGING_POM.equals(dependency.getType())) {
-                continue;
-            }
-            String coordinate = getCoordinate(dependency);
-            if (!jobCoordinateMap.containsKey(coordinate) && !talendLibCoordinateMap.containsKey(coordinate)) {
-                _3rdDepLibMap.put(coordinate, dependency);
-                addToDuplicateLibs(duplicateLibs, dependency);
-            }
-        }
+        IProcessor processor = getJobProcessor();
 
-        // add missing modules from the job generation of children
-        Set<ModuleNeeded> fullModulesList = new HashSet<>();
-        for (JobInfo jobInfo : childrenJobInfo) {
-            fullModulesList
-                    .addAll(LastGenerationInfo
-                            .getInstance()
-                            .getModulesNeededWithSubjobPerJob(jobInfo.getJobId(), jobInfo.getJobVersion()));
-        }
-        for (ModuleNeeded moduleNeeded : fullModulesList) {
-            if (moduleNeeded.isExcluded()) {
-                continue;
+        // current job
+        Property currentJobProperty = processor.getProperty();
+        jobCoordinate.add(getJobCoordinate(currentJobProperty));
+
+        // children jobs without test cases
+        Set<JobInfo> childrenJobInfo = !hasLoopDependency()
+                ? processor.getBuildChildrenJobs().stream().filter(j -> !j.isTestContainer()).collect(Collectors.toSet())
+                : Collections.emptySet();
+        childrenJobInfo.forEach(j -> jobCoordinate.add(getJobCoordinate(j.getProcessItem().getProperty())));
+
+        // talend libraries and codes
+        String projectGroupId = PomIdsHelper
+                .getProjectGroupId(ProjectManager.getInstance().getProject(currentJobProperty).getTechnicalLabel());
+
+        List<Dependency> dependencies = new ArrayList<>();
+        // codes
+        addCodesDependencies(dependencies);
+
+        // codes dependencies (optional)
+        ERepositoryObjectType.getAllTypesOfCodes().forEach(t -> dependencies.addAll(PomUtil.getCodesDependencies(t)));
+
+        // libraries of talend/3rd party
+        dependencies.addAll(processor.getNeededModules(TalendProcessOptionConstants.MODULES_EXCLUDE_SHADED).stream()
+                .filter(m -> !m.isExcluded()).map(m -> createDenpendency(m, false)).collect(Collectors.toSet()));
+
+        // missing modules from the job generation of children
+        childrenJobInfo.forEach(j -> dependencies
+                .addAll(LastGenerationInfo.getInstance().getModulesNeededPerJob(j.getJobId(), j.getJobVersion()).stream()
+                        .filter(m -> !m.isExcluded()).map(m -> createDenpendency(m, false)).collect(Collectors.toSet())));
+
+        Set<ModuleNeeded> modules = new HashSet<>();
+        // testcase modules from current job (optional)
+        modules.addAll(ProcessorDependenciesManager.getTestcaseNeededModules(currentJobProperty));
+
+        // testcase modules from children job (optional)
+        childrenJobInfo.forEach(
+                j -> modules.addAll(ProcessorDependenciesManager.getTestcaseNeededModules(j.getProcessItem().getProperty())));
+
+        dependencies
+                .addAll(modules.stream().filter(m -> !m.isExcluded()).map(m -> createDenpendency(m, true))
+                        .collect(Collectors.toSet()));
+
+        dependencies.stream().filter(d -> !MavenConstants.PACKAGING_POM.equals(d.getType())).forEach(d -> {
+            String coordinate = getCoordinate(d);
+            String groupId = d.getGroupId();
+            boolean optional = ((SortableDependency) d).isAssemblyOptional();
+            if (jobCoordinate.contains(coordinate) || talendLibCoordinate.contains(coordinate)
+                    || _3rdLibCoordinate.contains(coordinate)) {
+                return;
             }
-            MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(moduleNeeded.getMavenUri());
-            String coordinate = getCoordinate(artifact.getGroupId(), artifact.getArtifactId(), artifact.getType(),
-                    artifact.getVersion());
-            if (!jobCoordinateMap.containsKey(coordinate) && !talendLibCoordinateMap.containsKey(coordinate)
-                    && !_3rdDepLibMap.containsKey(coordinate)) {
-                Dependency dependencyObject = getDependencyObject(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion(), artifact.getType(), artifact.getClassifier());
-                if (MavenConstants.DEFAULT_LIB_GROUP_ID.equals(artifact.getGroupId())
-                        || artifact.getGroupId().startsWith(projectGroupId)) {
-                    talendLibCoordinateMap.put(coordinate, dependencyObject);
-                } else {
-                    _3rdDepLibMap.put(coordinate, dependencyObject);
-                    Dependency dependency = PomUtil
-                            .createDependency(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion(),
-                                    artifact.getType(), artifact.getClassifier());
-                    addToDuplicateLibs(duplicateLibs, dependency);
+            if (MavenConstants.DEFAULT_LIB_GROUP_ID.equals(groupId) || groupId.startsWith(projectGroupId)) {
+                if (!optional) {
+                    talendLibCoordinate.add(coordinate);
                 }
+            } else {
+                if (!optional) {
+                    _3rdLibCoordinate.add(coordinate);
+                }
+                addToDuplicateLibs(duplicateLibs, d);
             }
-        }
+        });
 
         Iterator<String> iterator = duplicateLibs.keySet().iterator();
         while (iterator.hasNext()) {
-            String key = iterator.next();
-            Set<Dependency> dupDependencies = duplicateLibs.get(key);
-            if (dupDependencies.size() < 2) {
-                // remove non-duplicated dependencies
+            Set<Dependency> dupDependencies = duplicateLibs.get(iterator.next());
+            if (dupDependencies.size() < 2 // remove unique dependency
+            /* || dupDependencies.stream().filter(d -> !((SortableDependency) d).isAssemblyOptional()).count() == 1 */) {
+                // remove when only one required dependencies, means others are from codes/testcase
+                // don't do this now at least it won't have problem in studio
+                // in some case, the needed jar is not in main job pom, maven will get the nearest one which could be
+                // wrong
                 iterator.remove();
             } else {
-                // remove duplicated dependencies from 3rd lib list
-                for (Dependency dependency : dupDependencies) {
-                    _3rdDepLibMap.remove(getCoordinate(dependency));
-                }
+                // remove duplicate dependencies from 3rd party libs
+                dupDependencies.stream().map(d -> getCoordinate(d)).forEach(c -> _3rdLibCoordinate.remove(c));
             }
         }
 
         try {
             Document document = PomUtil.loadAssemblyFile(null, assemblyFile);
-            // add talend libs & codes
-            setupDependencySetNode(document, talendLibCoordinateMap, "lib", "${artifact.artifactId}.${artifact.extension}",
-                    false, false);
-            // add 3rd party libs <dependencySet>
-            setupDependencySetNode(document, _3rdDepLibMap, "lib", null, false, false);
+            setupDependencySetNode(document, talendLibCoordinate, "lib", "${artifact.artifactId}.${artifact.extension}", false,
+                    false);
+            // add 3rd party libs
+            setupDependencySetNode(document, _3rdLibCoordinate, "lib", null, false, false);
+
             // add jobs
-            setupDependencySetNode(document, jobCoordinateMap, "${talend.job.name}",
+            setupDependencySetNode(document, jobCoordinate,
+                    "${talend.job.name}",
                     "${artifact.build.finalName}.${artifact.extension}", true, false);
             // add duplicate dependencies if exists
-            setupFileNode(document, duplicateLibs);
+            setupFileNode(document, duplicateLibs.values().stream().flatMap(s -> s.stream()).collect(Collectors.toSet()));
 
             PomUtil.saveAssemblyFile(assemblyFile, document);
         } catch (Exception e) {
@@ -760,43 +731,85 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         }
     }
 
+    // remove duplicate job dependencies and only keep the latest one
+    // keep high priority dependencies if set by tLibraryLoad
+    // FIXME not used now since tacokit component use specific dependency version. so we must include all job dependencies.
+    // but problem will remain for CI when need to download jars from nexus, maven will only resolve one of them.
+    private Set<Dependency> convertToDistinctedJobDependencies(String jobId, String jobVersion, Set<ModuleNeeded> neededModules) {
+        Set<Dependency> highPriorityDependencies = LastGenerationInfo.getInstance()
+                .getHighPriorityModuleNeededPerJob(jobId, jobVersion).stream().map(m -> createDenpendency(m, false))
+                .collect(Collectors.toSet());
+        Map<String, Dependency> highPriorityDependenciesMap = new HashMap<>();
+        highPriorityDependencies.forEach(d -> highPriorityDependenciesMap.putIfAbsent(getCheckDupCoordinate(d), d));
+
+        Set<Dependency> jobDependencies = neededModules.stream().filter(m -> !m.isExcluded())
+                .map(m -> createDenpendency(m, false)).collect(Collectors.toSet());
+        Map<String, Set<Dependency>> jobDependenciesMap = new HashMap<>();
+        jobDependencies.forEach(d -> {
+            String coordinate = getCheckDupCoordinate(d);
+            if (jobDependenciesMap.get(coordinate) == null) {
+                jobDependenciesMap.put(coordinate, new LinkedHashSet<>());
+            }
+            jobDependenciesMap.get(coordinate).add(d);
+        });
+
+        Set<Dependency> filteredDependencies = new HashSet<>();
+        jobDependenciesMap.forEach((key, value) -> {
+            Optional<Dependency> target = null;
+            if (highPriorityDependenciesMap.containsKey(key)) {
+                Dependency highPriorityDependency = highPriorityDependenciesMap.get(key);
+                target = value.stream().filter(d -> getCoordinate(highPriorityDependency).equals(getCoordinate(d))).findFirst();
+            } else {
+                target = value.stream().sorted(
+                        (d1, d2) -> new ComparableVersion(d2.getVersion()).compareTo(new ComparableVersion(d1.getVersion())))
+                        .findFirst();
+            }
+            if (target.isPresent()) {
+                filteredDependencies.add(target.get());
+            }
+        });
+
+        return filteredDependencies;
+    }
+
+    private Dependency createDenpendency(ModuleNeeded moduleNeeded, boolean optional) {
+        SortableDependency dependency = (SortableDependency) PomUtil.createModuleDependency(moduleNeeded.getMavenUri());
+        dependency.setAssemblyOptional(optional);
+        return dependency;
+    }
+
     private String getCoordinate(Dependency dependency) {
         return getCoordinate(dependency.getGroupId(), dependency.getArtifactId(), dependency.getType(),
-                dependency.getVersion());
+                dependency.getVersion(), dependency.getClassifier());
     }
 
-    protected String getCoordinate(String groupId, String artifactId, String type, String version) {
+    private String getCheckDupCoordinate(Dependency dependency) {
+        return getCoordinate(dependency.getGroupId(), dependency.getArtifactId(), dependency.getType(), null,
+                dependency.getClassifier());
+    }
+
+    protected String getJobCoordinate(Property property) {
+        return getCoordinate(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(property),
+                MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(property), null);
+    }
+
+    protected String getCoordinate(String groupId, String artifactId, String type, String version, String classifier) {
         String separator = ":"; //$NON-NLS-1$
-        String coordinate = groupId + separator;
-        coordinate += artifactId + separator;
-        if (type != null) {
-            coordinate += type;
+        String coordinate = groupId + separator + artifactId;
+        if (StringUtils.isNotBlank(type)) {
+            coordinate += separator + type;
+        }
+        if (StringUtils.isNotBlank(classifier)) {
+            coordinate += separator + classifier;
         }
 
-        if (version != null) {
+        if (StringUtils.isNotBlank(version)) {
             coordinate += separator + version;
         }
 
         return coordinate;
     }
 
-    protected String getAssemblyCoordinate(Dependency dependency) {
-        String separator = ":"; //$NON-NLS-1$
-        String coordinate = dependency.getGroupId() + separator;
-        coordinate += dependency.getArtifactId() + separator;
-        if (dependency.getType() != null) {
-            coordinate += dependency.getType();
-        }
-        if (dependency.getClassifier() != null) {
-            coordinate += separator + "*";
-        }
-        if (dependency.getVersion() != null) {
-            coordinate += separator + dependency.getVersion();
-        }
-
-        return coordinate;
-    }
-
     protected Dependency getDependencyObject(String groupId, String artifactId, String version, String type, String classifier) {
         Dependency object = new SortableDependency();
         object.setGroupId(groupId);
@@ -809,17 +822,15 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
     }
 
     private void addToDuplicateLibs(Map<String, Set<Dependency>> map, Dependency dependency) {
-        String coordinate =
-                getCoordinate(dependency.getGroupId(), dependency.getArtifactId(), dependency.getType(), null);
+        String coordinate = getCheckDupCoordinate(dependency);
         if (!map.containsKey(coordinate)) {
-            Set<Dependency> set = new HashSet<>();
-            map.put(coordinate, set);
+            map.put(coordinate, new HashSet<>());
         }
         map.get(coordinate).add(dependency);
     }
 
-    protected void setupDependencySetNode(Document document, Map<String, Dependency> libIncludes, String outputDir,
-            String fileNameMapping, boolean useProjectArtifact, boolean unpack) {
+    protected void setupDependencySetNode(Document document, Set<String> libIncludes, String outputDir, String fileNameMapping,
+            boolean useProjectArtifact, boolean unpack) {
         if (libIncludes.isEmpty()) {
             return;
         }
@@ -839,9 +850,9 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
         Node includesNode = document.createElement("includes");
         dependencySetNode.appendChild(includesNode);
 
-        for (Dependency dependency : libIncludes.values()) {
+        for (String coodinate : libIncludes) {
             Node includeNode = document.createElement("include");
-            includeNode.setTextContent(getAssemblyCoordinate(dependency));
+            includeNode.setTextContent(coodinate);
             includesNode.appendChild(includeNode);
         }
 
@@ -863,42 +874,77 @@ public class CreateMavenJobPom extends AbstractMavenProcessorPom {
 
     }
 
-    private void setupFileNode(Document document, Map<String, Set<Dependency>> duplicateDependencies) {
+    private void setupFileNode(Document document, Set<Dependency> duplicateDependencies) throws CoreException {
+        Node filesNode = document.getElementsByTagName("files").item(0);
+        // TESB-27614:NPE while building a route
+        if (filesNode == null) {
+            return;
+        }
         if (duplicateDependencies.isEmpty()) {
             return;
         }
-        try {
-            IMaven maven = MavenPlugin.getMaven();
-            ArtifactRepository repository = maven.getLocalRepository();
-            Node filesNode = document.getElementsByTagName("files").item(0);
-            for (Entry<String, Set<Dependency>> entry : duplicateDependencies.entrySet()) {
-                Set<Dependency> dependencies = entry.getValue();
-                for (Dependency dependency : dependencies) {
-                    String sourceLocation = maven
-                            .getArtifactPath(repository, dependency.getGroupId(), dependency.getArtifactId(),
-                                    dependency.getVersion(), dependency.getType(), dependency.getClassifier());
-                    Path path = new File(repository.getBasedir()).toPath().resolve(sourceLocation);
-                    sourceLocation = path.toString();
-                    String destName = path.getFileName().toString();
-                    Node fileNode = document.createElement("file");
-                    filesNode.appendChild(fileNode);
-
-                    Node sourcesNode = document.createElement("source");
-                    sourcesNode.setTextContent(sourceLocation);
-                    fileNode.appendChild(sourcesNode);
-
-                    Node outputDirNode = document.createElement("outputDirectory");
-                    outputDirNode.setTextContent("lib");
-                    fileNode.appendChild(outputDirNode);
-
-                    Node destNameNode = document.createElement("destName");
-                    destNameNode.setTextContent(destName);
-                    fileNode.appendChild(destNameNode);
-                }
+        IMaven maven = MavenPlugin.getMaven();
+        ArtifactRepository repository = maven.getLocalRepository();
+        for (Dependency dependency : duplicateDependencies) {
+            if (((SortableDependency) dependency).isAssemblyOptional()) {
+                continue;
             }
-        } catch (CoreException e) {
-            ExceptionHandler.process(e);
+            String sourceLocation = maven.getArtifactPath(repository, dependency.getGroupId(), dependency.getArtifactId(),
+                    dependency.getVersion(), dependency.getType(), dependency.getClassifier());
+            Path path = new File(repository.getBasedir()).toPath().resolve(sourceLocation);
+            sourceLocation = path.toString();
+            String destName = path.getFileName().toString();
+            Node fileNode = document.createElement("file");
+            filesNode.appendChild(fileNode);
+
+            Node sourcesNode = document.createElement("source");
+            sourcesNode.setTextContent(sourceLocation);
+            fileNode.appendChild(sourcesNode);
+
+            Node outputDirNode = document.createElement("outputDirectory");
+            outputDirNode.setTextContent("lib");
+            fileNode.appendChild(outputDirNode);
+
+            Node destNameNode = document.createElement("destName");
+            destNameNode.setTextContent(destName);
+            fileNode.appendChild(destNameNode);
         }
     }
 
+    protected Plugin addSkipDockerMavenPlugin() {
+        Plugin plugin = new Plugin();
+
+        plugin.setGroupId("io.fabric8");
+        plugin.setArtifactId("fabric8-maven-plugin");
+        plugin.setVersion("4.0.0");
+
+        Xpp3Dom skip = new Xpp3Dom("skip");
+        // skip.setValue("${docker.skip}");
+        skip.setValue("true");
+
+        Xpp3Dom configuration = new Xpp3Dom("configuration");
+        configuration.addChild(skip);
+
+        List<PluginExecution> pluginExecutions = new ArrayList<PluginExecution>();
+        PluginExecution pluginExecutionStart = new PluginExecution();
+        pluginExecutionStart.setId("start");
+        pluginExecutionStart.setPhase("none");
+        pluginExecutionStart.setConfiguration(configuration);
+
+        pluginExecutions.add(pluginExecutionStart);
+
+        PluginExecution pluginExecutionPushImage = new PluginExecution();
+        pluginExecutionPushImage.setId("push-image");
+        pluginExecutionPushImage.setPhase("none");
+        pluginExecutionPushImage.setConfiguration(configuration);
+
+        pluginExecutions.add(pluginExecutionPushImage);
+
+        plugin.setExecutions(pluginExecutions);
+        plugin.setConfiguration(configuration);
+
+        return plugin;
+
+    }
+
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenJobletPom.java

@@ -123,7 +123,9 @@ public class CreateMavenJobletPom extends AbstractMavenProcessorPom {
     @Override
     protected void addDependencies(Model model) {
         try {
-            getProcessorDependenciesManager().updateDependencies(null, model);
+            if (ERepositoryObjectType.JOBLET == ERepositoryObjectType.getType(getJobProcessor().getProperty())) {
+                getProcessorDependenciesManager().updateDependencies(null, model);
+            }
             addChildrenDependencies(model.getDependencies());
         } catch (ProcessorException e) {
             ExceptionHandler.process(e);

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/creator/CreateMavenStandardJobOSGiPom.java

@@ -18,12 +18,12 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import org.apache.maven.model.Build;
-import org.apache.maven.model.Dependency;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.Plugin;
 import org.apache.maven.model.PluginExecution;
@@ -33,11 +33,18 @@ import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.exception.PersistenceException;
+import org.talend.core.GlobalServiceRegister;
+import org.talend.core.IESBService;
 import org.talend.core.PluginChecker;
 import org.talend.core.model.process.JobInfo;
+import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.relationship.Relation;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
+import org.talend.core.model.repository.ERepositoryObjectType;
+import org.talend.core.model.repository.IRepositoryViewObject;
+import org.talend.core.repository.model.ProxyRepositoryFactory;
 import org.talend.core.runtime.maven.MavenConstants;
 import org.talend.core.runtime.process.ITalendProcessJavaProject;
 import org.talend.core.runtime.process.TalendProcessArgumentConstant;
@@ -50,6 +57,7 @@ import org.talend.designer.maven.tools.AggregatorPomsHelper;
 import org.talend.designer.maven.utils.PomIdsHelper;
 import org.talend.designer.maven.utils.PomUtil;
 import org.talend.designer.runprocess.IProcessor;
+import org.talend.repository.model.IProxyRepositoryFactory;
 import org.talend.utils.io.FilesUtils;
 import org.w3c.dom.Document;
 
@@ -108,8 +116,9 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
     @Override
     protected Model createModel() {
         Model model = super.createModel();
-
+        
         boolean isServiceOperation = isServiceOperation(getJobProcessor().getProperty());
+
         List<Profile> profiles = model.getProfiles();
 
         for (Profile profile : profiles) {
@@ -141,12 +150,25 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
         }
         model.setName(model.getName() + " Bundle");
         model.addProperty("talend.job.finalName", "${talend.job.name}-bundle-${project.version}");
+        Build build = model.getBuild();
+        
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IESBService.class)) {
+            IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
+
+            if (isServiceOperation || service.isRESTService((ProcessItem) getJobProcessor().getProperty().getItem())
+                    || isRouteOperation(getJobProcessor().getProperty())) {
+                build.addPlugin(addSkipDockerMavenPlugin());
+            }
+        }
+        
         if (isServiceOperation) {
             model.addProperty("cloud.publisher.skip", "true");
-            Build build = model.getBuild();
+            
+            build = model.getBuild();
 
             List<Plugin> removePlugins = new ArrayList<Plugin>();
             if (build != null) {
+
                 List<Plugin> plugins = build.getPlugins();
                 for (Plugin p : plugins) {
                     if (p.getArtifactId().equals("maven-deploy-plugin")) {
@@ -203,31 +225,41 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
 
     @Override
     protected void updateDependencySet(IFile assemblyFile) {
-        Map<String, Dependency> jobCoordinateMap = new HashMap<String, Dependency>();
+        Set<String> jobCoordinate = new HashSet<>();
         if (!hasLoopDependency()) {
             // add children jobs
             Set<JobInfo> childrenJobInfo = getJobProcessor().getBuildChildrenJobs();
             for (JobInfo jobInfo : childrenJobInfo) {
                 Property property = jobInfo.getProcessItem().getProperty();
-                String coordinate = getCoordinate(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo),
-                        MavenConstants.PACKAGING_JAR, PomIdsHelper.getJobVersion(property));
-                Dependency dependency = getDependencyObject(PomIdsHelper.getJobGroupId(property), PomIdsHelper.getJobArtifactId(jobInfo), PomIdsHelper.getJobVersion(property),
-                                MavenConstants.PACKAGING_JAR, null);
-                jobCoordinateMap.put(coordinate, dependency);
+
+                String bundle = "";
+
+                if (!jobInfo.isJoblet()) {
+                    property = jobInfo.getProcessItem().getProperty();
+                    if ("OSGI".equals(property.getAdditionalProperties().get(TalendProcessArgumentConstant.ARG_BUILD_TYPE))) {
+                        bundle = "-bundle";
+                    }
+                }
+
+                String coordinate = getCoordinate(
+                		PomIdsHelper.getJobGroupId(property), 
+                		PomIdsHelper.getJobArtifactId(jobInfo) + bundle,
+                        MavenConstants.PACKAGING_JAR, 
+                        PomIdsHelper.getJobVersion(property), 
+                        null);
+
+                jobCoordinate.add(coordinate);
             }
         }
-        // add parent job
-        Property parentProperty = this.getJobProcessor().getProperty();
-        String parentCoordinate = getCoordinate(PomIdsHelper.getJobGroupId(parentProperty),
-                PomIdsHelper.getJobArtifactId(parentProperty), MavenConstants.PACKAGING_JAR,
-                PomIdsHelper.getJobVersion(parentProperty));
-        Dependency parentDependency = getDependencyObject(PomIdsHelper.getJobGroupId(parentProperty), PomIdsHelper.getJobArtifactId(parentProperty), PomIdsHelper.getJobVersion(parentProperty),
-                        MavenConstants.PACKAGING_JAR, null);
-        jobCoordinateMap.put(parentCoordinate, parentDependency);
+        // add current job
+        Property currentJobProperty = getJobProcessor().getProperty();
+        String parentCoordinate = getJobCoordinate(currentJobProperty);
+        jobCoordinate.add(parentCoordinate);
         try {
             Document document = PomUtil.loadAssemblyFile(null, assemblyFile);
             // add jobs
-            setupDependencySetNode(document, jobCoordinateMap, null, "${artifact.build.finalName}.${artifact.extension}", true,
+            setupDependencySetNode(document, jobCoordinate, null, "${artifact.build.finalName}.${artifact.extension}",
+                    true,
                     true);
             PomUtil.saveAssemblyFile(assemblyFile, document);
         } catch (Exception e) {
@@ -260,14 +292,64 @@ public class CreateMavenStandardJobOSGiPom extends CreateMavenJobPom {
      * @return
      */
     public boolean isServiceOperation(Property property) {
-        List<Relation> relations = RelationshipItemBuilder.getInstance().getItemsRelatedTo(property.getId(),
-                property.getVersion(), RelationshipItemBuilder.JOB_RELATION);
+        List<IRepositoryViewObject> serviceRepoList = null;
 
-        for (Relation relation : relations) {
-            if (RelationshipItemBuilder.SERVICES_RELATION.equals(relation.getType())) {
-                return true;
+        boolean isDataServiceOperation = false;
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IESBService.class)) {
+            IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
+            try {
+                IProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
+                serviceRepoList = factory.getAll(ERepositoryObjectType.valueOf(ERepositoryObjectType.class, "SERVICES"));
+
+                for (IRepositoryViewObject serviceItem : serviceRepoList) {
+                    if (service != null) {
+                        List<String> jobIds = service.getSerivceRelatedJobIds(serviceItem.getProperty().getItem());
+                        if (jobIds.contains(property.getId())) {
+                            isDataServiceOperation = true;
+                            break;
+                        }
+                    }
+                }
+
+            } catch (PersistenceException e) {
+                ExceptionHandler.process(e);
             }
         }
-        return false;
+
+        return isDataServiceOperation;
+    }
+
+    public boolean isRouteOperation(Property property) {
+        List<IRepositoryViewObject> routeRepoList = null;
+
+        boolean isRouteOperation = false;
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IESBService.class)) {
+
+            IESBService service = (IESBService) GlobalServiceRegister.getDefault().getService(IESBService.class);
+
+            try {
+                IProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
+                routeRepoList = factory.getAll(ERepositoryObjectType.valueOf(ERepositoryObjectType.class, "ROUTE"));
+
+                for (IRepositoryViewObject routeItem : routeRepoList) {
+                    if (service != null) {
+
+                        List<Relation> relations = RelationshipItemBuilder.getInstance().getItemsRelatedTo(routeItem.getId(),
+                                routeItem.getVersion(), RelationshipItemBuilder.JOB_RELATION);
+                        for (Relation relation : relations) {
+                            if (relation.getType() == RelationshipItemBuilder.JOB_RELATION) {
+                                if (relation.getId().equals(property.getId())) {
+                                    isRouteOperation = true;
+                                }
+                            }
+                        }
+                    }
+                }
+
+            } catch (PersistenceException e) {
+                ExceptionHandler.process(e);
+            }
+        }
+        return isRouteOperation;
     }
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/PomIdsHelper.java

@@ -260,6 +260,12 @@ public class PomIdsHelper {
         return manager.getValue(MavenConstants.POM_FILTER);
     }
 
+    public static boolean useProfileModule() {
+        String projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
+        ProjectPreferenceManager manager = getPreferenceManager(projectTechName);
+        return manager.getBoolean(MavenConstants.USE_PROFILE_MODULE);
+    }
+
     private static String getGroupId(String projectTechName, String baseName, Property property) {
         if (projectTechName == null) {
             projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/PomUtil.java

@@ -25,15 +25,16 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.security.SecureRandom;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -81,6 +82,7 @@ import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.relationship.Relation;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
+import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.nexus.TalendMavenResolver;
@@ -96,6 +98,7 @@ import org.talend.designer.maven.template.MavenTemplateManager;
 import org.talend.designer.maven.tools.AggregatorPomsHelper;
 import org.talend.designer.maven.tools.ProcessorDependenciesManager;
 import org.talend.designer.runprocess.IProcessor;
+import org.talend.designer.runprocess.IRunProcessService;
 import org.talend.repository.ProjectManager;
 import org.talend.utils.xml.XmlUtils;
 import org.w3c.dom.Attr;
@@ -136,6 +139,7 @@ public class PomUtil {
          * copied the codes from createMavenModel of MavenModelManager
          */
         ByteArrayOutputStream buf = new ByteArrayOutputStream();
+        sortModules(model);
         MavenPlugin.getMaven().writeModel(model, buf);
 
         ByteArrayInputStream source = new ByteArrayInputStream(buf.toByteArray());
@@ -157,6 +161,7 @@ public class PomUtil {
         properties.putAll(model.getProperties());
         model.setProperties(properties);
         ByteArrayOutputStream buf = new ByteArrayOutputStream();
+        sortModules(model);
         MavenPlugin.getMaven().writeModel(model, buf);
 
         ByteArrayInputStream source = new ByteArrayInputStream(buf.toByteArray());
@@ -178,6 +183,17 @@ public class PomUtil {
         }
     }
 
+    public static void sortModules(Model model) {
+        if (model != null) {
+            List<String> modules = model.getModules();
+            if (modules != null && !modules.isEmpty()) {
+                List<String> sortedModules = new LinkedList<>(modules);
+                Collections.sort(sortedModules);
+                model.setModules(sortedModules);
+            }
+        }
+    }
+
     private static void safeClose(Closeable stream) {
         try {
             if (stream != null) {
@@ -511,6 +527,10 @@ public class PomUtil {
         return repoPath + "/" + artifactPath; //$NON-NLS-1$
     }
 
+    public static String getLocalRepositoryPath() {
+        return MavenPlugin.getMaven().getLocalRepositoryPath();
+    }
+
     /**
      * Get absolute path for installed artifact
      *
@@ -613,7 +633,9 @@ public class PomUtil {
         File pomFile = new File(baseFolder, TalendMavenConstants.POM_FILE_NAME);
 
         ByteArrayOutputStream buf = new ByteArrayOutputStream();
-        MavenPlugin.getMaven().writeModel(createModel(artifact), buf);
+        Model model = createModel(artifact);
+        sortModules(model);
+        MavenPlugin.getMaven().writeModel(model, buf);
 
         DocumentBuilderFactory documentBuilderFactory = XmlUtils.getSecureDocumentBuilderFactory();
         documentBuilderFactory.setNamespaceAware(false);
@@ -676,37 +698,20 @@ public class PomUtil {
      *
      * in order to make sure no compile error for editor, so add all needed dependencies always.
      */
-    public static Collection<Dependency> getCodesDependencies(IFile projectPomFile, String projectTechName) throws CoreException {
-        Map<String, Dependency> codesDependencies = new LinkedHashMap<>();
-
-        // routines
-        addCodeDependencies(codesDependencies, projectPomFile, TalendMavenConstants.DEFAULT_ROUTINES_ARTIFACT_ID,
-                MavenTemplateManager.getRoutinesTempalteModel(projectTechName));
-
-        // beans
-        addCodeDependencies(codesDependencies, projectPomFile, TalendMavenConstants.DEFAULT_BEANS_ARTIFACT_ID,
-                MavenTemplateManager.getBeansTempalteModel(projectTechName));
-        // pigudfs
-        addCodeDependencies(codesDependencies, projectPomFile, TalendMavenConstants.DEFAULT_PIGUDFS_ARTIFACT_ID,
-                MavenTemplateManager.getPigUDFsTempalteModel(projectTechName));
-
-        return codesDependencies.values();
-    }
-
-    private static void addCodeDependencies(Map<String, Dependency> codesDependencies, IFile projectPomFile, String pomName,
-            Model defaultModel) throws CoreException {
-        IFile routinesPomFile = projectPomFile.getProject().getFile(PomUtil.getPomFileName(pomName));
-        Model model = defaultModel;
-        if (routinesPomFile.exists()) {
-            model = MODEL_MANAGER.readMavenModel(routinesPomFile);
-        }
-        List<Dependency> dependencies = model.getDependencies();
-        for (Dependency d : dependencies) {
-            String mvnUrl = generateMvnUrl(d);
-            if (!codesDependencies.containsKey(mvnUrl)) {
-                codesDependencies.put(mvnUrl, d);
+    public static Set<Dependency> getCodesDependencies(ERepositoryObjectType codeType) {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
+            IRunProcessService runProcessService = GlobalServiceRegister.getDefault().getService(IRunProcessService.class);
+            try {
+                Model model = MODEL_MANAGER.readMavenModel(runProcessService.getTalendCodeJavaProject(codeType).getProjectPom());
+                return model.getDependencies().stream().map(
+                        d -> createDependency(d.getGroupId(), d.getArtifactId(), d.getVersion(), d.getType(), d.getClassifier()))
+                        .peek(d -> ((SortableDependency) d).setAssemblyOptional(true)).collect(Collectors.toSet());
+            } catch (CoreException e) {
+                ExceptionHandler.process(e);
             }
         }
+        return Collections.emptySet();
+
     }
 
     public static List<String> getMavenCodesModules(IProcess process) {
@@ -1127,4 +1132,20 @@ public class PomUtil {
                     || pathname.getName().equals("m2e-lastUpdated.properties"); //$NON-NLS-1$
         }
     };
+
+    public static boolean checkIfJobDependencyExist(IFile parentJobPom, Property property) throws Exception {
+        boolean found = false;
+        Model model = MavenPlugin.getMavenModelManager().readMavenModel(parentJobPom);
+        String groupId = PomIdsHelper.getJobGroupId(property);
+        String artifactId = PomIdsHelper.getJobArtifactId(property);
+        String version = PomIdsHelper.getJobVersion(property);
+        for (Dependency dependency : model.getDependencies()) {
+            if (dependency.getGroupId().equals(groupId) && dependency.getArtifactId().equals(artifactId)
+                    && dependency.getVersion().equals(version)) {
+                found = true;
+                break;
+            }
+        }
+        return found;
+    }
 }

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/SortableDependency.java

@@ -1,13 +1,79 @@
 package org.talend.designer.maven.utils;
 
+import org.apache.maven.artifact.versioning.ComparableVersion;
 import org.apache.maven.model.Dependency;
 
 public class SortableDependency extends Dependency implements Comparable<SortableDependency> {
 
+    private static final long serialVersionUID = -6295226523517981508L;
+
+    /**
+     * dependencies in testcase/routines/beans no need to be assembled in final zip.
+     */
+    private boolean isAssemblyOptional;
+
     @Override
     public int compareTo(SortableDependency o) {
 
-        return getArtifactId().compareTo(o.getArtifactId());
+        int compare = getArtifactId().compareTo(o.getArtifactId());
+        if (compare == 0) {
+            // FIXME according to Maven official Doc for dependencies:
+            // https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html
+            // when pom has duplicate version of dependencies, Maven should take the first one
+            // but in practice(maven 3.5.3 embedded), it always take the last one
+            // we only need job dependencies in final job zip rather than testcase's
+            // so we always put the latest job dependency at the bottom
+            // if maven fix it in future, we need to reverse the order as well.
+            if (isAssemblyOptional && !o.isAssemblyOptional) {
+                return -1;
+            }
+            if (!isAssemblyOptional && o.isAssemblyOptional) {
+                return 1;
+            }
+            return new ComparableVersion(getVersion()).compareTo(new ComparableVersion(o.getVersion()));
+        }
+        return compare;
     }
 
+    public boolean isAssemblyOptional() {
+        return isAssemblyOptional;
+    }
+
+    public void setAssemblyOptional(boolean isAssemblyOptional) {
+        this.isAssemblyOptional = isAssemblyOptional;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 17;
+        result = 31 * result + (getGroupId() == null ? 0 : getGroupId().hashCode());
+        result = 31 * result + (getArtifactId() == null ? 0 : getArtifactId().hashCode());
+        result = 31 * result + (getVersion() == null ? 0 : getVersion().hashCode());
+        result = 31 * result + (getType() == null ? 0 : getType().hashCode());
+        result = 31 * result + (getClassifier() == null ? 0 : getClassifier().hashCode());
+        result = 31 * result + (isAssemblyOptional() ? 1231 : 1237);
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (!(obj instanceof SortableDependency)) {
+            return false;
+        }
+        SortableDependency caseobj = (SortableDependency) obj;
+        return (getGroupId() == caseobj.getGroupId() || (getGroupId() != null && getGroupId().equals(caseobj.getGroupId())))
+                && (getArtifactId() == caseobj.getArtifactId()
+                        || (getArtifactId() != null && getArtifactId().equals(caseobj.getArtifactId())))
+                && (getVersion() == caseobj.getVersion() || (getVersion() != null && getVersion().equals(caseobj.getVersion())))
+                && (getType() == caseobj.getType() || (getType() != null && getType().equals(caseobj.getType())))
+                && (getClassifier() == caseobj.getClassifier()
+                        || (getClassifier() != null && getClassifier().equals(caseobj.getClassifier())))
+                && (isAssemblyOptional() == caseobj.isAssemblyOptional());
+    }
 }

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/utils/RemoteModulesHelper.java

@@ -34,6 +34,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
@@ -207,6 +208,7 @@ public class RemoteModulesHelper {
                     for (String groupId : snapshotgroupIds) {
                         List<MavenArtifact> searchResults = customerRepHandler.search(groupId, null, null, false, true);
                         monitor.worked(10);
+                        replaceTimeStampWithSnapShot(searchResults);
                         addModulesToCache(mavenUristoSearch, searchResults, localCache);
                     }
                 }
@@ -216,6 +218,24 @@ public class RemoteModulesHelper {
             }
         }
 
+        private void replaceTimeStampWithSnapShot(List<MavenArtifact> searchResults) {
+            if (searchResults != null && searchResults.size() > 0) {
+                for (MavenArtifact m : searchResults) {
+                    String version = m.getVersion();
+                    if (version != null && !version.contains(MavenConstants.SNAPSHOT)
+                            && timeStampRegex(version)) {
+                        version = version.substring(0, version.indexOf("-")) + MavenConstants.SNAPSHOT;
+                        m.setVersion(version);
+                    }
+                }
+            }
+        }
+
+        private boolean timeStampRegex(String version) {
+            boolean match = Pattern.matches("\\S*[-]{1}[0-9]{8}[.]{1}[0-9]{6}[-]{1}\\S*", version);
+            return match;
+        }
+
         private void searchFromLocalDataFile(Set<String> mavenUristoSearch, IProgressMonitor monitor) {
             LibraryDataService service = LibraryDataService.getInstance();
             List<MavenArtifact> artifactList = new ArrayList<MavenArtifact>();

main/plugins/org.talend.librariesmanager/src/main/java/messages.properties

@@ -26,5 +26,6 @@ ModulesNeededProvider.Module.Exception=Needed module list do not contains this m
 
 ShareLibsJob.message=Share libraries from maven to {0} server
 ShareLibsJob.sharingLibraries=Sharing libraries : {0}
+ShareLibsJob.monitor.cancelled=User cancelled.
 
 ExtensionModuleManager.moduleGroup.cycleReference=Module group {0} was cycle referenced!

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/maven/ShareLibrareisHelper.java

@@ -13,26 +13,36 @@
 package org.talend.librariesmanager.maven;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.codehaus.jackson.map.util.ISO8601Utils;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Status;
 import org.eclipse.core.runtime.SubMonitor;
 import org.eclipse.core.runtime.jobs.Job;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.model.general.ModuleNeeded;
-import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.RepositoryArtifactHandlerManager;
 import org.talend.core.nexus.TalendLibsServerManager;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.librariesmanager.i18n.Messages;
+import org.talend.librariesmanager.model.service.LocalLibraryManager;
+import org.talend.librariesmanager.nexus.utils.VersionUtil;
 
 /**
  * created by Talend on 2015年7月31日 Detailled comment
@@ -49,9 +59,7 @@ public abstract class ShareLibrareisHelper {
         IStatus status = Status.OK_STATUS;
         // deploy to maven if needed and share to custom nexus
         try {
-            int searchLimit = 50;
             setJobName(job, Messages.getString("ShareLibsJob.message", TYPE_NEXUS));
-            final List<MavenArtifact> searchResults = new ArrayList<MavenArtifact>();
             ArtifactRepositoryBean customNexusServer = TalendLibsServerManager.getInstance().getCustomNexusServer();
             IRepositoryArtifactHandler customerRepHandler = RepositoryArtifactHandlerManager
                     .getRepositoryHandler(customNexusServer);
@@ -60,69 +68,110 @@ public abstract class ShareLibrareisHelper {
                 if (filesToShare == null) {
                     return Status.CANCEL_STATUS;
                 }
-                SubMonitor mainSubMonitor = SubMonitor.convert(monitor, filesToShare.size());
 
                 // collect groupId to search
                 Set<String> groupIds = new HashSet<String>();
+                Map<String, List<MavenArtifact>> snapshotArtifactMap = new HashMap<String, List<MavenArtifact>>();
+                Map<String, List<MavenArtifact>> releaseArtifactMap = new HashMap<String, List<MavenArtifact>>();
+                Set<String> snapshotGroupIdSet = new HashSet<String>();
+                Set<String> releaseGroupIdSet = new HashSet<String>();
                 for (ModuleNeeded module : filesToShare.keySet()) {
+                    checkCancel(monitor);
                     if (module.getMavenUri() != null) {
                         MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(module.getMavenUri());
                         if (parseMvnUrl != null) {
                             groupIds.add(parseMvnUrl.getGroupId());
+                            if (isSnapshotVersion(parseMvnUrl.getVersion())) {
+                                snapshotGroupIdSet.add(parseMvnUrl.getGroupId());
+                            } else {
+                                releaseGroupIdSet.add(parseMvnUrl.getGroupId());
+                            }
                         }
                     }
                 }
+                List<MavenArtifact> searchResults = new ArrayList<MavenArtifact>();
                 for (String groupId : groupIds) {
-                    searchResults.addAll(customerRepHandler.search(groupId, null, null, true, true));
+                    checkCancel(monitor);
+                    if (releaseGroupIdSet.contains(groupId)) {
+                        searchResults = customerRepHandler.search(groupId, null, null, true, false);
+                        if (searchResults != null) {
+                            for (MavenArtifact result : searchResults) {
+                                checkCancel(monitor);
+                                putArtifactToMap(result, releaseArtifactMap, false);
+                            }
+                        }
+                    }
+                    checkCancel(monitor);
+                    if (snapshotGroupIdSet.contains(groupId)) {
+                        searchResults = customerRepHandler.search(groupId, null, null, false, true);
+                        if (searchResults != null) {
+                            for (MavenArtifact result : searchResults) {
+                                checkCancel(monitor);
+                                putArtifactToMap(result, snapshotArtifactMap, true);
+                            }
+                        }
+                    }
                 }
-
-                int limit = searchLimit;
-                int shareIndex = 0;
                 Iterator<ModuleNeeded> iterator = filesToShare.keySet().iterator();
+                Map<File, MavenArtifact> shareFiles = new HashMap<>();
                 while (iterator.hasNext()) {
-                    if (monitor.isCanceled()) {
-                        return Status.CANCEL_STATUS;
-                    }
-                    shareIndex++;
-                    if (shareIndex == limit) {
-                        limit += searchLimit;
-                    }
-
+                    checkCancel(monitor);
                     ModuleNeeded next = iterator.next();
                     File file = filesToShare.get(next);
-                    String name = file.getName();
                     MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(next.getMavenUri());
                     if (artifact == null) {
                         continue;
                     }
-
-                    boolean eixst = false;
-                    String groupId = artifact.getGroupId();
-                    String artifactId = artifact.getArtifactId();
-                    String version = artifact.getVersion();
-                    for (MavenArtifact remoteAtifact : searchResults) {
-                        String rGroup = remoteAtifact.getGroupId();
-                        String rArtifact = remoteAtifact.getArtifactId();
-                        String rVersion = remoteAtifact.getVersion();
-                        if (groupId != null && artifactId != null && version != null && groupId.equals(rGroup)
-                                && artifactId.equals(rArtifact) && version.equals(rVersion)) {
-                            eixst = true;
-                            break;
+                    // If from custom component definition file
+                    if (LocalLibraryManager.isSystemCacheFile(file.getName())
+                            || (LocalLibraryManager.isComponentDefinitionFileType(file.getName())
+                                    && isTalendLibraryGroupId(artifact))) {
+                        continue;
+                    }
+                    try {
+                        Integer.parseInt(artifact.getType());
+                        // FIXME unexpected type if it's an integer, should fix it in component module definition.
+                        continue;
+                    } catch (NumberFormatException e) {
+                        //
+                    }
+                    boolean isSnapshotVersion = isSnapshotVersion(artifact.getVersion());
+                    String key = getArtifactKey(artifact, isSnapshotVersion);
+                    List<MavenArtifact> artifactList = null;
+                    if (isSnapshotVersion) {
+                        artifactList = snapshotArtifactMap.get(key);
+                    } else {
+                        artifactList = releaseArtifactMap.get(key);
+                        // skip checksum for release artifact.
+                        if (artifactList != null && artifactList.contains(artifact)
+                                && !Boolean.getBoolean("force_libs_release_update")) {
+                            continue;
                         }
                     }
-                    if (eixst) {
-                        continue;
+                    if (artifactList != null && artifactList.size() > 0) {
+                        if (isSameFileWithRemote(file, artifactList, customNexusServer, customerRepHandler, isSnapshotVersion)) {
+                            continue;
+                        }
                     }
-                    mainSubMonitor.setTaskName(Messages.getString("ShareLibsJob.sharingLibraries", name));
-
+                    shareFiles.put(file, artifact);
+                }
+                SubMonitor mainSubMonitor = SubMonitor.convert(monitor, shareFiles.size());
+                for (Map.Entry<File, MavenArtifact> entry : shareFiles.entrySet()) {
+                    checkCancel(monitor);
                     try {
-                        shareToRepository(file, artifact);
+                        File k = entry.getKey();
+                        MavenArtifact v = entry.getValue();
+                        mainSubMonitor.setTaskName(Messages.getString("ShareLibsJob.sharingLibraries", k.getName()));
+                        shareToRepository(k, v);
                         mainSubMonitor.worked(1);
                     } catch (Exception e) {
-                        continue;
+                        ExceptionHandler.process(e);
                     }
                 }
             }
+        } catch (InterruptedException e) {
+            ExceptionHandler.process(e);
+            status = Status.CANCEL_STATUS;
         } catch (Exception e) {
             status = new Status(IStatus.ERROR, "unknown", IStatus.ERROR, "Share libraries failed !", e);
         }
@@ -131,12 +180,126 @@ public abstract class ShareLibrareisHelper {
 
     }
 
+    private boolean isTalendLibraryGroupId(MavenArtifact artifact) {
+        if ("org.talend.libraries".equalsIgnoreCase(artifact.getGroupId())) {
+            return true;
+        }
+        return false;
+    }
+
+    public void putArtifactToMap(MavenArtifact artifact, Map<String, List<MavenArtifact>> map, boolean isShapshot) {
+        String key = getArtifactKey(artifact, isShapshot);
+        List<MavenArtifact> list = map.get(key);
+        if (list == null) {
+            list = new ArrayList<MavenArtifact>();
+            map.put(key, list);
+        }
+        list.add(artifact);
+    }
+
+    private String getArtifactKey(MavenArtifact artifact, boolean isShapshot) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(artifact.getGroupId()).append("-");
+        sb.append(artifact.getArtifactId()).append("-");
+        String version = artifact.getVersion();
+        if (isShapshot) {
+            version = VersionUtil.getSNAPSHOTVersion(version);
+        }
+        sb.append(version);
+        if (StringUtils.isNotEmpty(artifact.getClassifier())) {
+            sb.append("-").append(artifact.getClassifier());
+        }
+        return sb.toString();
+    }
+
+    private boolean isSameFileWithRemote(File localFile, List<MavenArtifact> artifactList,
+            ArtifactRepositoryBean customNexusServer, IRepositoryArtifactHandler customerRepHandler, boolean isSnapshotVersion)
+            throws Exception {
+        String localFileShaCode = DigestUtils.shaHex(new FileInputStream(localFile));
+        String remoteSha1 = null;
+        if (ArtifactRepositoryBean.NexusType.ARTIFACTORY.name().equalsIgnoreCase(customNexusServer.getType())) {
+            MavenArtifact lastUpdatedArtifact = getLateUpdatedMavenArtifact(artifactList);
+            if (lastUpdatedArtifact != null) {
+                remoteSha1 = lastUpdatedArtifact.getSha1();
+            }
+        } else if (ArtifactRepositoryBean.NexusType.NEXUS_3.name().equalsIgnoreCase(customNexusServer.getType())) {
+            MavenArtifact lastUpdatedArtifact = artifactList.stream().max(Comparator.comparing(e -> e.getVersion())).get();
+            if (lastUpdatedArtifact != null) {
+                remoteSha1 = lastUpdatedArtifact.getSha1();
+            }
+        } else {
+            if (!isSnapshotVersion && !Boolean.getBoolean("force_libs_release_update")) {
+                return true;
+            }
+            MavenArtifact lastUpdatedArtifact = artifactList.get(0);
+            if (lastUpdatedArtifact != null) {
+                remoteSha1 = customerRepHandler.resolveRemoteSha1(lastUpdatedArtifact, !isSnapshotVersion);
+            }
+        }
+        if (StringUtils.equals(localFileShaCode, remoteSha1)) {
+            return true;
+        }
+        return false;
+    }
+
+    private MavenArtifact getLateUpdatedMavenArtifact(List<MavenArtifact> artifactList) {
+        if (artifactList.size() == 1) {
+            return artifactList.get(0);
+        }
+        MavenArtifact latestVersion = null;
+        Date lastUpdate = null;
+        for (MavenArtifact art : artifactList) {
+            if (latestVersion == null) {
+                if (art.getLastUpdated() != null) {
+                    latestVersion = art;
+                    lastUpdate = parsetDate(art.getLastUpdated());
+                }
+            } else if (art.getLastUpdated() != null && lastUpdate != null) {
+                Date artLastUpdate = parsetDate(art.getLastUpdated());
+                if (artLastUpdate != null && lastUpdate.getTime() < artLastUpdate.getTime()) {
+                    latestVersion = art;
+                    lastUpdate = artLastUpdate;
+                }
+            }
+        }
+        if (latestVersion != null) {
+            return latestVersion;
+        } else {
+            return artifactList.get(artifactList.size() - 1);
+        }
+    }
+
+    private Date parsetDate(String strDate) {
+        Date date = null;
+        if (strDate != null) {
+            try {
+                date = ISO8601Utils.parse(strDate);
+            } catch (Exception ex) {
+                ExceptionHandler.process(ex);
+            }
+        }
+        return date;
+    }
+
+    private boolean isSnapshotVersion(String version) {
+        if (version != null && version.toUpperCase().endsWith(MavenUrlHelper.VERSION_SNAPSHOT)) {
+            return true;
+        }
+        return false;
+    }
+
     private void setJobName(Job job, String jobName) {
         if (job != null) {
             job.setName(jobName);
         }
     }
 
+    protected void checkCancel(IProgressMonitor monitor) throws InterruptedException {
+        if (monitor.isCanceled()) {
+            throw new InterruptedException(Messages.getString("ShareLibsJob.monitor.cancelled"));
+        }
+    }
+
     public abstract Map<ModuleNeeded, File> getFilesToShare(IProgressMonitor monitor);
 
     public abstract void shareToRepository(File jarFile, MavenArtifact module) throws Exception;

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/ModulesNeededProvider.java

@@ -706,8 +706,13 @@ public class ModulesNeededProvider {
                 ModuleNeeded toAdd = new ModuleNeeded(context, currentImport.getMODULE(), currentImport.getMESSAGE(),
                         isRequired);
                 toAdd.setMavenUri(currentImport.getMVN());
-                if (!isRequired && "BeanItem".equals(routine.eClass().getName())) {
-                	toAdd.getExtraAttributes().put("IS_OSGI_EXCLUDED", Boolean.TRUE);
+                if (!isRequired) {
+                    if ("BeanItem".equals(routine.eClass().getName())) {
+                        toAdd.getExtraAttributes().put("IS_OSGI_EXCLUDED", Boolean.TRUE);
+                    }
+                    if ("RoutineItem".equals(routine.eClass().getName())) {
+                        continue;
+                    }
                 }
                 // toAdd.setStatus(ELibraryInstallStatus.INSTALLED);
                 importNeedsList.add(toAdd);

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/LocalLibraryManager.java

@@ -113,6 +113,18 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
 
     private MavenArtifactsHandler deployer;
 
+    private static final List<String> COMPONENT_DEFINITION_FILE_TYPE_LIST = new ArrayList<String>() {
+
+        {
+            add(".javajet");
+            add(".png");
+            add(".jpg");
+            add("_java.xml");
+            add(".properties");
+            add(".txt");
+        }
+    };
+
     /**
      * DOC nrousseau LocalLibraryManager constructor comment.
      */
@@ -224,7 +236,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         }
     }
 
-    private void guessMavenRUIFromIndex(File jarFile, Map<String, String> sourceAndMavenUri) {
+    public void guessMavenRUIFromIndex(File jarFile, Map<String, String> sourceAndMavenUri) {
         // TODO????? should deploy with all versions
         String urisFromIndex = LibrariesIndexManager.getInstance().getMavenLibIndex().getJarsToRelativePath()
                 .get(jarFile.getName());
@@ -379,6 +391,24 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         return false;
     }
 
+    public static boolean isComponentDefinitionFileType(String fileName) {
+        if (fileName != null) {
+            for (String type : COMPONENT_DEFINITION_FILE_TYPE_LIST) {
+                if (fileName.toLowerCase().endsWith(type)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    public static boolean isSystemCacheFile(String fileName) {
+        if ("Thumbs.db".equals(fileName)) {
+            return true;
+        }
+        return false;
+    }
+
     private File retrieveJarFromLocal(ModuleNeeded module) {
         File jarFile = null;
         Set<String> toResolve = guessMavenURI(module);
@@ -1224,7 +1254,10 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
 
         saveMavenIndex(mavenURIMap, monitorWrap);
         savePlatfromURLIndex(platformURLMap, monitorWrap);
-
+        
+        if (service != null) {
+            deployLibsFromCustomComponents(service, platformURLMap);
+        }
     }
 
     /**
@@ -1293,6 +1326,41 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
                 continue;
             }
         }
+    }
+
+    private void deployLibsFromCustomComponents(IComponentsService service, Map<String, String> platformURLMap) {
+        Set<File> needToDeploy = new HashSet<>();
+        List<ComponentProviderInfo> componentsFolders = service.getComponentsFactory().getComponentsProvidersInfo();
+        for (ComponentProviderInfo providerInfo : componentsFolders) {
+            String id = providerInfo.getId();
+            try {
+                File file = new File(providerInfo.getLocation());
+                if ("org.talend.designer.components.model.UserComponentsProvider".equals(id)
+                        || "org.talend.designer.components.exchange.ExchangeComponentsProvider".equals(id)) {
+                    if (file.isDirectory()) {
+                        List<File> jarFiles = FilesUtils.getJarFilesFromFolder(file, null);
+                        if (jarFiles.size() > 0) {
+                            for (File jarFile : jarFiles) {
+                                String name = jarFile.getName();
+                                if (!canDeployFromCustomComponentFolder(name)
+                                        || platformURLMap.get(name) != null) {
+                                    continue;
+                                }
+                                needToDeploy.add(jarFile);
+                            }
+                        }
+                    } else {
+                        if (platformURLMap.get(file.getName()) != null) {
+                            continue;
+                        }
+                        needToDeploy.add(file);
+                    }
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+                continue;
+            }
+        }
 
         // deploy needed jars for User and Exchange component providers
         if (!needToDeploy.isEmpty()) {
@@ -1328,6 +1396,13 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         }
     }
 
+    private boolean canDeployFromCustomComponentFolder(String fileName) {
+        if (isSystemCacheFile(fileName) || isComponentDefinitionFileType(fileName)) {
+            return false;
+        }
+        return true;
+    }
+
     private void warnDuplicated(List<ModuleNeeded> modules, Set<String> duplicates, String type) {
         for (String lib : duplicates) {
             Set<String> components = new HashSet<>();
@@ -1594,7 +1669,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         return isSameFile(jarFile, jarInLib);
     }
 
-    private boolean isSameFile(File f1, File f2) {
+    public boolean isSameFile(File f1, File f2) {
         if (f1 == null || f2 == null) {
             return false;
         }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/AbstractArtifactRepositoryHandler.java

@@ -13,13 +13,18 @@
 package org.talend.librariesmanager.nexus;
 
 import java.io.File;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.util.Dictionary;
 import java.util.Hashtable;
 
+import org.apache.commons.lang.StringUtils;
 import org.talend.core.nexus.ArtifactRepositoryBean;
 import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.NexusConstants;
 import org.talend.core.nexus.TalendMavenResolver;
+import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.utils.string.StringUtilities;
 
 /**
@@ -62,10 +67,22 @@ public abstract class AbstractArtifactRepositoryHandler implements IRepositoryAr
         if (props == null) {
             props = new Hashtable<String, String>();
         }
-        String repositories = null;
-        String custom_server = serverBean.getServer();
+
         String custom_user = serverBean.getUserName();
         String custom_pass = serverBean.getPassword();
+        try {
+            if (StringUtils.isNotBlank(custom_user)) {
+                custom_user = URLEncoder.encode(custom_user, StandardCharsets.UTF_8.toString());
+            }
+            if (StringUtils.isNotBlank(custom_pass)) {
+                custom_pass = URLEncoder.encode(custom_pass, StandardCharsets.UTF_8.toString());
+            }
+        } catch (UnsupportedEncodingException e1) {
+            throw new RuntimeException(e1);
+        }
+
+        String repositories = null;
+        String custom_server = serverBean.getServer();
         String release_rep = serverBean.getRepositoryId();
         String snapshot_rep = serverBean.getSnapshotRepId();
         if (custom_server.endsWith(NexusConstants.SLASH)) {
@@ -161,4 +178,10 @@ public abstract class AbstractArtifactRepositoryHandler implements IRepositoryAr
 
     @Override
     public abstract IRepositoryArtifactHandler clone();
+    
+    @Override
+    public String resolveRemoteSha1(MavenArtifact artifact, boolean fromRelease) throws Exception {
+        return artifact.getSha1();
+    }
+        
 }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/ArtifacoryRepositoryHandler.java

@@ -14,6 +14,7 @@ package org.talend.librariesmanager.nexus;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -25,11 +26,15 @@ import org.apache.http.client.fluent.Request;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.params.CoreConnectionPNames;
 import org.apache.http.util.EntityUtils;
 import org.eclipse.m2e.core.MavenPlugin;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.TalendProxySelector.IProxySelectorProvider;
+import org.talend.core.nexus.HttpClientTransport;
 import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.NexusConstants;
+import org.talend.core.nexus.NexusServerUtils;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.designer.maven.aether.RepositorySystemFactory;
@@ -43,9 +48,9 @@ import net.sf.json.JSONObject;
  */
 public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandler {
 
-    private String SEARCH_SERVICE = "api/search/gavc?";
+    private String SEARCH_SERVICE = "api/search/gavc?"; //$NON-NLS-1$
 
-    private String SEARCH_RESULT_PREFIX = "api/storage/";
+    private String SEARCH_RESULT_PREFIX = "api/storage/";//$NON-NLS-1$
 
     /*
      * (non-Javadoc)
@@ -80,17 +85,30 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
     }
 
     private boolean doConnectionCheck(String repositoryUrl) throws ClientProtocolException, IOException {
-        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword();
-        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes()));
-        Header authority = new BasicHeader("Authorization", basicAuth);
+        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword(); //$NON-NLS-1$
+        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes())); //$NON-NLS-1$
+        Header authority = new BasicHeader("Authorization", basicAuth); //$NON-NLS-1$
         HttpGet get = new HttpGet(repositoryUrl);
         get.addHeader(authority);
         DefaultHttpClient httpclient = new DefaultHttpClient();
-        HttpResponse response = httpclient.execute(get);
-        if (response.getStatusLine().getStatusCode() == 200) {
-            return true;
+        httpclient.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, NexusServerUtils.getTimeout());
+        httpclient.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, NexusServerUtils.getTimeout());
+        IProxySelectorProvider proxySelector = null;
+        try {
+            try {
+                proxySelector = HttpClientTransport.addProxy(httpclient, new URI(repositoryUrl));
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+            HttpResponse response = httpclient.execute(get);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                return true;
+            }
+            return false;
+        } finally {
+            HttpClientTransport.removeProxy(proxySelector);
+            httpclient.getConnectionManager().shutdown();
         }
-        return false;
     }
 
     /*
@@ -103,51 +121,53 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
     public List<MavenArtifact> search(String groupIdToSearch, String artifactId, String versionToSearch, boolean fromRelease,
             boolean fromSnapshot) throws Exception {
         String serverUrl = serverBean.getServer();
-        if (!serverUrl.endsWith("/")) {
-            serverUrl = serverUrl + "/";
+        if (!serverUrl.endsWith("/")) { //$NON-NLS-1$
+            serverUrl = serverUrl + "/"; //$NON-NLS-1$
         }
         String searchUrl = serverUrl + SEARCH_SERVICE;
 
-        String repositoryId = "";
+        String repositoryId = ""; //$NON-NLS-1$
         if (fromRelease) {
             repositoryId = serverBean.getRepositoryId();
         }
         if (fromSnapshot) {
-            if ("".equals(repositoryId)) {
+            if ("".equals(repositoryId)) { //$NON-NLS-1$
                 repositoryId = serverBean.getSnapshotRepId();
             } else {
-                repositoryId = repositoryId + "," + serverBean.getSnapshotRepId();
+                repositoryId = repositoryId + "," + serverBean.getSnapshotRepId(); //$NON-NLS-1$
             }
         }
         String query = "";//$NON-NLS-1$
-        if (!"".equals(repositoryId)) {
+        if (!"".equals(repositoryId)) { //$NON-NLS-1$
             query = "repos=" + repositoryId;//$NON-NLS-1$
         }
         if (groupIdToSearch != null) {
-            if (!"".equals(query)) {
-                query = query + "&";
+            if (!"".equals(query)) { //$NON-NLS-1$
+                query = query + "&"; //$NON-NLS-1$
             }
             query = query + "g=" + groupIdToSearch;//$NON-NLS-1$
         }
         if (artifactId != null) {
-            if (!"".equals(query)) {
-                query = query + "&";
+            if (!"".equals(query)) { //$NON-NLS-1$
+                query = query + "&"; //$NON-NLS-1$
             }
             query = query + "a=" + artifactId;//$NON-NLS-1$
         }
 
         if (versionToSearch != null) {
-            if (!"".equals(query)) {
-                query = query + "&";
+            if (!"".equals(query)) { //$NON-NLS-1$
+                query = query + "&"; //$NON-NLS-1$
             }
             query = query + "v=" + versionToSearch;//$NON-NLS-1$
         }
         searchUrl = searchUrl + query;
         Request request = Request.Get(searchUrl);
-        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword();
-        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes()));
-        Header authority = new BasicHeader("Authorization", basicAuth);
+        String userPass = serverBean.getUserName() + ":" + serverBean.getPassword(); //$NON-NLS-1$
+        String basicAuth = "Basic " + new String(new Base64().encode(userPass.getBytes())); //$NON-NLS-1$
+        Header authority = new BasicHeader("Authorization", basicAuth); //$NON-NLS-1$
         request.addHeader(authority);
+        Header resultDetailHeader = new BasicHeader("X-Result-Detail", "info"); //$NON-NLS-1$ //$NON-NLS-2$
+        request.addHeader(resultDetailHeader);
         List<MavenArtifact> resultList = new ArrayList<MavenArtifact>();
 
         HttpResponse response = request.execute().returnResponse();
@@ -155,11 +175,11 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
         if (content.isEmpty()) {
             return resultList;
         }
-        JSONObject responseObject = new JSONObject().fromObject(content);
-        String resultStr = responseObject.getString("results");
+        JSONObject responseObject = JSONObject.fromObject(content);
+        String resultStr = responseObject.getString("results"); //$NON-NLS-1$
         JSONArray resultArray = null;
         try {
-            resultArray = new JSONArray().fromObject(resultStr);
+            resultArray = JSONArray.fromObject(resultStr);
         } catch (Exception e) {
             throw new Exception(resultStr);
         }
@@ -167,46 +187,57 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
             String resultUrl = serverUrl + SEARCH_RESULT_PREFIX;
             for (int i = 0; i < resultArray.size(); i++) {
                 JSONObject jsonObject = resultArray.getJSONObject(i);
-                String uri = jsonObject.getString("uri");
+                String lastUpdated = jsonObject.getString("lastUpdated"); //$NON-NLS-1$
+                String uri = jsonObject.getString("uri"); //$NON-NLS-1$
                 uri = uri.substring(resultUrl.length(), uri.length());
-                String[] split = uri.split("/");
+                String[] split = uri.split("/"); //$NON-NLS-1$
                 if (split.length > 4) {
                     String fileName = split[split.length - 1];
-                    if (!fileName.endsWith("pom")) {
+                    if (!fileName.endsWith("pom")) { //$NON-NLS-1$
                         String type = null;
                         int dotIndex = fileName.lastIndexOf('.');
                         if (dotIndex > 0) {
                             type = fileName.substring(dotIndex + 1);
-
                         }
                         if (type != null) {
                             MavenArtifact artifact = new MavenArtifact();
                             String v = split[split.length - 2];
                             String a = split[split.length - 3];
-                            String g = "";
+                            String g = ""; //$NON-NLS-1$
                             for (int j = 1; j < split.length - 3; j++) {
-                                if ("".equals(g)) {
+                                if ("".equals(g)) { //$NON-NLS-1$
                                     g = split[j];
                                 } else {
-                                    g = g + "." + split[j];
+                                    g = g + "." + split[j]; //$NON-NLS-1$
                                 }
                             }
                             artifact.setGroupId(g);
                             artifact.setArtifactId(a);
                             artifact.setVersion(v);
                             artifact.setType(type);
+                            artifact.setLastUpdated(lastUpdated);
+                            fillChecksumData(jsonObject, artifact);
                             resultList.add(artifact);
                         }
                     }
-
                 }
-
             }
         }
-
         return resultList;
     }
 
+    private void fillChecksumData(JSONObject responseObject, MavenArtifact artifact) {
+        if (responseObject.containsKey("checksums")) { //$NON-NLS-1$
+            JSONObject checkSum = responseObject.getJSONObject("checksums"); //$NON-NLS-1$
+            if (checkSum != null && checkSum.containsKey("sha1")) { //$NON-NLS-1$
+                artifact.setSha1(checkSum.getString("sha1")); //$NON-NLS-1$
+            }
+            if (checkSum != null && checkSum.containsKey("md5")) { //$NON-NLS-1$
+                artifact.setMd5(checkSum.getString("md5")); //$NON-NLS-1$
+            }
+        }
+    }
+
     /*
      * (non-Javadoc)
      *
@@ -216,7 +247,7 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
     @Override
     public void deploy(File content, String groupId, String artifactId, String classifier, String extension, String version)
             throws Exception {
-        String repositoryId = "";
+        String repositoryId = ""; //$NON-NLS-1$
         boolean isRelease = !version.endsWith(MavenUrlHelper.VERSION_SNAPSHOT);
         if (isRelease) {
             repositoryId = serverBean.getRepositoryId();
@@ -238,7 +269,7 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
     @Override
     public void deployWithPOM(File content, File pomFile, String groupId, String artifactId, String classifier, String extension,
             String version) throws Exception {
-        String repositoryId = "";
+        String repositoryId = ""; //$NON-NLS-1$
         boolean isRelease = !version.endsWith(MavenUrlHelper.VERSION_SNAPSHOT);
         if (isRelease) {
             repositoryId = serverBean.getRepositoryId();

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/Nexus2RepositoryHandler.java

@@ -137,5 +137,11 @@ public class Nexus2RepositoryHandler extends AbstractArtifactRepositoryHandler {
         RepositorySystemFactory.deployWithPOM(content, pomFile, localRepository, repositoryId, repositoryurl,
                 serverBean.getUserName(), serverBean.getPassword(), groupId, artifactId, classifier, extension, version);
     }
-
+    
+    @Override
+    public String resolveRemoteSha1(MavenArtifact artifact, boolean fromRelease) throws Exception {
+        return NexusServerUtils.resolveSha1(serverBean.getServer(), serverBean.getUserName(), serverBean.getPassword(),
+                fromRelease ? serverBean.getRepositoryId() : serverBean.getSnapshotRepId(), artifact.getGroupId(),
+                artifact.getArtifactId(), artifact.getVersion(), artifact.getType());
+    }
 }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/Nexus3RepositoryHandler.java

@@ -14,8 +14,11 @@ package org.talend.librariesmanager.nexus;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.http.Header;
@@ -24,10 +27,15 @@ import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.params.CoreConnectionPNames;
 import org.apache.log4j.Logger;
 import org.eclipse.m2e.core.MavenPlugin;
 import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.TalendProxySelector.IProxySelectorProvider;
+import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.nexus.HttpClientTransport;
 import org.talend.core.nexus.IRepositoryArtifactHandler;
+import org.talend.core.nexus.NexusServerUtils;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.designer.maven.aether.RepositorySystemFactory;
@@ -48,6 +56,8 @@ public class Nexus3RepositoryHandler extends AbstractArtifactRepositoryHandler {
 
     private INexus3SearchHandler currentQueryHandler = null;
 
+    private static final Map<ArtifactRepositoryBean, INexus3SearchHandler> LAST_HANDLER_MAP = new HashMap<ArtifactRepositoryBean, INexus3SearchHandler>();
+    
     private static List<INexus3SearchHandler> queryHandlerList = new ArrayList<INexus3SearchHandler>();
 
     @Override
@@ -94,11 +104,25 @@ public class Nexus3RepositoryHandler extends AbstractArtifactRepositoryHandler {
         HttpGet get = new HttpGet(repositoryUrl);
         get.addHeader(authority);
         DefaultHttpClient httpclient = new DefaultHttpClient();
-        HttpResponse response = httpclient.execute(get);
-        if (response.getStatusLine().getStatusCode() == 200) {
-            return true;
+        
+        httpclient.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, NexusServerUtils.getTimeout());
+        httpclient.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, NexusServerUtils.getTimeout());
+        IProxySelectorProvider proxySelector = null;
+        try {
+            try {
+                proxySelector = HttpClientTransport.addProxy(httpclient, new URI(repositoryUrl));
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+            HttpResponse response = httpclient.execute(get);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                return true;
+            }
+            return false;
+        } finally {
+            HttpClientTransport.removeProxy(proxySelector);
+            httpclient.getConnectionManager().shutdown();
         }
-        return false;
     }
 
     /*
@@ -124,8 +148,12 @@ public class Nexus3RepositoryHandler extends AbstractArtifactRepositoryHandler {
 
     private List<MavenArtifact> doSearch(String repositoryId, String groupIdToSearch, String artifactId, String versionToSearch)
             throws Exception {
+        initQueryHandler();
         if (currentQueryHandler == null) {
-            currentQueryHandler = getQueryHandler();
+            currentQueryHandler = LAST_HANDLER_MAP.get(serverBean);
+            if (currentQueryHandler == null) {
+                currentQueryHandler = queryHandlerList.get(0);
+            }
         }
         List<MavenArtifact> result = new ArrayList<MavenArtifact>();
         try {
@@ -146,16 +174,16 @@ public class Nexus3RepositoryHandler extends AbstractArtifactRepositoryHandler {
                 }
             }
         }
+        LAST_HANDLER_MAP.put(serverBean, currentQueryHandler);
         return result;
     }
 
-    private INexus3SearchHandler getQueryHandler() {
+    private void initQueryHandler() {
         if (queryHandlerList.size() == 0) {
             queryHandlerList.add(new Nexus3V1SearchHandler(serverBean));
             queryHandlerList.add(new Nexus3BetaSearchHandler(serverBean));
             queryHandlerList.add(new Nexus3ScriptSearchHandler(serverBean));
         }
-        return queryHandlerList.get(0);
     }
 
     /*

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/nexus3/handler/AbsNexus3SearchHandler.java

@@ -14,11 +14,13 @@ package org.talend.librariesmanager.nexus.nexus3.handler;
 // ============================================================================
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpHeaders;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.methods.HttpGet;
@@ -60,6 +62,7 @@ public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
 
     protected abstract String getSearchUrl();
 
+    @Override
     public List<MavenArtifact> search(String repositoryId, String groupIdToSearch, String artifactId, String versionToSearch)
             throws Exception {
         List<MavenArtifact> resultList = new ArrayList<MavenArtifact>();
@@ -84,6 +87,13 @@ public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
             protected HttpResponse execute(IProgressMonitor monitor, DefaultHttpClient httpClient, URI targetURI)
                     throws Exception {
                 HttpGet httpGet = new HttpGet(targetURI);
+                String userName = serverBean.getUserName();
+                if (StringUtils.isNotBlank(userName) && !Boolean.getBoolean("talend.studio.search.nexus3.disableBasicAuth")) {
+                    String auth = userName + ":" + serverBean.getPassword();
+                    String authHeader = "Basic " + new String(Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)));
+                    httpGet.setHeader(HttpHeaders.AUTHORIZATION, authHeader);
+                    httpClient.setCredentialsProvider(null);
+                }
                 HttpResponse response = httpClient.execute(httpGet);
                 if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
                     sb.append(EntityUtils.toString(response.getEntity()));
@@ -120,8 +130,9 @@ public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
                 artifact.setGroupId(jsonObject.getString("group")); //$NON-NLS-1$
                 artifact.setArtifactId(jsonObject.getString("name")); //$NON-NLS-1$
                 artifact.setVersion(jsonObject.getString("version")); //$NON-NLS-1$
-                JSONArray assertsArray = jsonObject.getJSONArray("assets"); //$NON-NLS-1$
+                JSONArray assertsArray = jsonObject.getJSONArray("assets"); //$NON-NLS-1$                
                 artifact.setType(getPackageType(assertsArray));
+                fillCheckSumData(assertsArray, artifact);
                 resultList.add(artifact);
             }
         }
@@ -135,6 +146,9 @@ public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
             for (int i = 0; i < assertsArray.size(); i++) {
                 JSONObject jsonObject = assertsArray.getJSONObject(i);
                 String path = jsonObject.getString("path"); //$NON-NLS-1$
+                if (path != null && path.endsWith(".exe")) { //$NON-NLS-1$
+                    return "exe"; //$NON-NLS-1$
+                }
                 if (path != null && path.endsWith(".zip")) { //$NON-NLS-1$
                     return "zip"; //$NON-NLS-1$
                 }
@@ -149,6 +163,27 @@ public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
         return type;
     }
 
+    private void fillCheckSumData(JSONArray assertsArray, MavenArtifact artifact) {
+        if (assertsArray != null) {
+            for (int i = 0; i < assertsArray.size(); i++) {
+                JSONObject jsonObject = assertsArray.getJSONObject(i);
+                if (jsonObject.containsKey("path")) { //$NON-NLS-1$
+                    String path = jsonObject.getString("path"); //$NON-NLS-1$
+                    if (path != null && path.endsWith(artifact.getType())) { // $NON-NLS-1$
+                        if (jsonObject.containsKey("checksum")) {
+                            JSONObject checksumObject = jsonObject.getJSONObject("checksum"); //$NON-NLS-1$
+                            if (checksumObject != null && checksumObject.containsKey("sha1")) {//$NON-NLS-1$
+                                artifact.setSha1(checksumObject.getString("sha1")); //$NON-NLS-1$
+                                artifact.setMd5(checksumObject.getString("md5")); //$NON-NLS-1$
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     protected String getQueryParameter(String repositoryId, String groupIdToSearch, String artifactId, String versionToSearch,
             String continuationToken) {
         StringBuffer sb = new StringBuffer();

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/utils/VersionUtil.java

@@ -0,0 +1,32 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.librariesmanager.nexus.utils;
+
+import org.talend.core.runtime.maven.MavenUrlHelper;
+
+/**
+ * DOC hwang  class global comment. Detailled comment
+ */
+public class VersionUtil {
+    
+    public static String getSNAPSHOTVersion(String rVersion) {
+        if(rVersion == null) {
+            return rVersion;
+        }
+        if(rVersion.contains("-")) {
+            return rVersion.substring(0, rVersion.indexOf("-") + 1) + MavenUrlHelper.VERSION_SNAPSHOT;
+        }
+        return rVersion;
+    }
+
+}

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/editor/MetadataTalendTypeEditor.java

@@ -72,6 +72,8 @@ import org.talend.core.model.metadata.MetadataTalendType;
 import org.talend.core.model.utils.ResourceModelHelper;
 import org.talend.core.model.utils.XSDValidater;
 import org.talend.core.repository.model.ProxyRepositoryFactory;
+import org.talend.core.runtime.CoreRuntimePlugin;
+import org.talend.core.runtime.projectsetting.ProjectPreferenceManager;
 import org.talend.metadata.managment.ui.MetadataManagmentUiPlugin;
 import org.talend.metadata.managment.ui.dialog.MappingFileCheckViewerDialog;
 import org.talend.metadata.managment.ui.i18n.Messages;
@@ -543,12 +545,12 @@ public class MetadataTalendTypeEditor extends FieldEditor {
             if (info != null) {
                 String id = null;
                 String infoName = info.fileName;
-                for (int i = 0; i < allDbms.length; i++) {
-                    if (allDbms[i].getLabel().equalsIgnoreCase(infoName.substring(0, infoName.indexOf(".")).replace("_", " "))) { //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-                        id = allDbms[i].getId();
+                for (Dbms allDbm : allDbms) {
+                    if (allDbm.getLabel().equalsIgnoreCase(infoName.substring(0, infoName.indexOf(".")).replace("_", " "))) { //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+                        id = allDbm.getId();
                     }
                 }
-                setSelectId(id); //$NON-NLS-1$
+                setSelectId(id); 
             }
         }
     }
@@ -569,7 +571,7 @@ public class MetadataTalendTypeEditor extends FieldEditor {
         if (confirm) {
             ICoreService coreService = null;
             if (GlobalServiceRegister.getDefault().isServiceRegistered(ICoreService.class)) {
-                coreService = (ICoreService) GlobalServiceRegister.getDefault().getService(ICoreService.class);
+                coreService = GlobalServiceRegister.getDefault().getService(ICoreService.class);
                 coreService.syncMappingsFileFromSystemToProject();
                 tmpFileManager.reload();
             }
@@ -686,6 +688,9 @@ public class MetadataTalendTypeEditor extends FieldEditor {
         }
         if (needReload) {
             tmpFileManager.reload();
+            ProjectPreferenceManager manager = CoreRuntimePlugin.getInstance().getProjectPreferenceManager();
+            manager.setValue(MetadataTalendType.UPDATED_MAPPING_FILES, true);
+            manager.save();
         }
     }
 

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/utils/ConnectionContextHelper.java

@@ -764,7 +764,25 @@ public final class ConnectionContextHelper {
                     Connection hadoopClusterConnection = hadoopClusterItem.getConnection();
                     ContextItem hadoopClusterContextItem = ContextUtils.getContextItemById2(hadoopClusterConnection
                             .getContextId());
-                    Set<String> hcNeededVars = retrieveContextVar(elementParameters, hadoopClusterConnection, category, true);
+                    Set<String> hcNeededVars = retrieveContextVar(elementParameters, hadoopClusterConnection, category, false);
+                    // The tSparkConfiguration & tHadoopConfManager are singleton in process.
+                    List<INode> nodesOfType = new ArrayList<INode>();
+                    List<? extends INode> spakconfNodes = process.getNodesOfType("tSparkConfiguration"); //$NON-NLS-1$
+                    if (spakconfNodes != null && !spakconfNodes.isEmpty()) {
+                        nodesOfType.addAll(spakconfNodes);
+                    }
+                    List<? extends INode> hadoopconfNodes = process.getNodesOfType("tHadoopConfManager"); //$NON-NLS-1$
+                    if (hadoopconfNodes != null && !hadoopconfNodes.isEmpty()) {
+                        nodesOfType.addAll(hadoopconfNodes);
+                    }
+                    for (INode node : nodesOfType) {
+                        Set<String> envirNeededVars = retrieveContextVar(node.getElementParameters(), hadoopClusterConnection,
+                                category, false);
+                        if (envirNeededVars != null && !envirNeededVars.isEmpty()) {
+                            hcNeededVars.addAll(envirNeededVars);
+                        }
+                    }
+
                     List<ContextItem> contextItems = new ArrayList<>();
                     if (contextItem != null || hadoopClusterContextItem != null) {
                         // find added variables

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/utils/ExtendedNodeConnectionContextUtils.java

@@ -125,11 +125,16 @@ public class ExtendedNodeConnectionContextUtils {
         QuboleS3BucketKey,
         QuboleS3Region,
 
+        // Override hadoop configuration
+        setHadoopConf,
+        hadoopConfSpecificJar,
+
         // DataBricks
         DataBricksEndpoint,
         DataBricksClusterId,
         DataBricksToken,
         DataBricksDBFSDepFolder
+
     }
 
     static List<IContextParameter> getContextVariables(final String prefixName, Connection conn, Set<IConnParamName> paramSet) {

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/ExtractMetaDataFromDataBase.java

@@ -42,6 +42,7 @@ import org.talend.commons.utils.workbench.extensions.IExtensionPointLimiter;
 import org.talend.core.database.EDatabase4DriverClassName;
 import org.talend.core.database.EDatabaseTypeName;
 import org.talend.core.database.conn.template.EDatabaseConnTemplate;
+import org.talend.core.database.conn.version.EDatabaseVersion4Drivers;
 import org.talend.core.model.metadata.IMetadataConnection;
 import org.talend.core.model.metadata.IMetadataTable;
 import org.talend.core.model.metadata.builder.ConvertionHelper;
@@ -334,12 +335,16 @@ public class ExtractMetaDataFromDataBase {
                     return connectionStatus;
                 }
             }
-            if (EDatabaseTypeName.SYBASEASE == EDatabaseTypeName.getTypeFromDisplayName(dbType)) {
-                boolean exsitedSybaseDB = checkSybaseDB(connection, sidOrDatabase);
-                if (!exsitedSybaseDB) {
-                    connectionStatus.setMessageException(
-                            Messages.getString("ExtractMetaDataFromDataBase.DatabaseNoPresent", sidOrDatabase)); //$NON-NLS-1$
-                    return connectionStatus;
+            String property = System.getProperty("disableCheckSybase16");//$NON-NLS-1$
+            Boolean disableCheckSybase16 = StringUtils.isEmpty(property) ? false : Boolean.valueOf(property);
+            if (!disableCheckSybase16) {
+                if (StringUtils.equals(EDatabaseVersion4Drivers.SYBASEIQ_16.getVersionValue(), dbVersionString)) {
+                    boolean exsitedSybaseDB = checkSybaseDB(connection, sidOrDatabase);
+                    if (!exsitedSybaseDB) {
+                        connectionStatus.setMessageException(
+                                Messages.getString("ExtractMetaDataFromDataBase.DatabaseNoPresent", sidOrDatabase)); //$NON-NLS-1$
+                        return connectionStatus;
+                    }
                 }
             }
 

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/ExtractMetaDataUtils.java

@@ -50,6 +50,7 @@ import org.talend.commons.utils.database.EXASOLDatabaseMetaData;
 import org.talend.commons.utils.database.JtdsDatabaseMetadata;
 import org.talend.commons.utils.database.SAPHanaDataBaseMetadata;
 import org.talend.commons.utils.database.SASDataBaseMetadata;
+import org.talend.commons.utils.database.Sybase16SADatabaseMetaData;
 import org.talend.commons.utils.database.SybaseDatabaseMetaData;
 import org.talend.commons.utils.database.SybaseIQDatabaseMetaData;
 import org.talend.commons.utils.database.TeradataDataBaseMetadata;
@@ -79,6 +80,7 @@ import org.talend.core.utils.TalendQuoteUtils;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.metadata.managment.connection.manager.HiveConnectionManager;
 import org.talend.metadata.managment.hive.EmbeddedHiveDataBaseMetadata;
+import org.talend.metadata.managment.utils.MetadataConnectionUtils;
 import org.talend.repository.ProjectManager;
 import org.talend.utils.exceptions.MissingDriverException;
 import org.talend.utils.sql.ConnectionUtils;
@@ -271,7 +273,10 @@ public class ExtractMetaDataUtils {
                     if (dbMetaData != null
                             && EDatabaseTypeName.SYBASEIQ.getDisplayName().equals(dbMetaData.getDatabaseProductName())) {
                         dbMetaData = createSybaseIQFakeDatabaseMetaData(conn);
-                    } else {
+                    } else if(dbMetaData != null
+                            && EDatabaseTypeName.SYBASEASE_SA.getDisplayName().equals(dbMetaData.getDatabaseProductName())) {
+                        dbMetaData = createSybase16SAFakeDatabaseMetaData(conn);
+                    }else {
                         dbMetaData = createSybaseFakeDatabaseMetaData(conn);
                     }
                 } else if (EDatabaseTypeName.HIVE.getDisplayName().equals(dbType) && isHiveEmbeddedConn(conn)) {
@@ -422,6 +427,11 @@ public class ExtractMetaDataUtils {
         SybaseDatabaseMetaData dmd = new SybaseDatabaseMetaData(conn);
         return dmd;
     }
+    
+    private DatabaseMetaData createSybase16SAFakeDatabaseMetaData(Connection conn) throws SQLException {
+        SybaseDatabaseMetaData dmd = new Sybase16SADatabaseMetaData(conn);
+        return dmd;
+    }
 
     private DatabaseMetaData createSybaseIQFakeDatabaseMetaData(Connection conn) throws SQLException {
         SybaseIQDatabaseMetaData dmd = new SybaseIQDatabaseMetaData(conn);
@@ -1042,6 +1052,9 @@ public class ExtractMetaDataUtils {
                         }
                     }else{
                         String jarName = librairesManagerService.getJarNameFromMavenuri(driverJarPathArg);
+                        if (jarName == null) {
+                            jarName = driverJarPathArg.split("/")[1] + ".jar";
+                        }
                         if (!new File(getJavaLibPath() + jarName).exists()) {
                             librairesManagerService.retrieve(jarName, getJavaLibPath(), new NullProgressMonitor());
                         }

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/metadata/managment/hive/HiveClassLoaderFactory.java

@@ -14,15 +14,19 @@ package org.talend.metadata.managment.hive;
 
 import java.io.File;
 import java.util.Set;
+import java.util.function.Consumer;
 
+import org.apache.commons.lang3.StringUtils;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.Platform;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.ILibraryManagerService;
 import org.talend.core.classloader.ClassLoaderFactory;
 import org.talend.core.classloader.DynamicClassLoader;
 import org.talend.core.database.conn.ConnParameterKeys;
 import org.talend.core.hadoop.EHadoopConfigurationJars;
+import org.talend.core.hadoop.HadoopConfJarBean;
 import org.talend.core.hadoop.IHadoopClusterService;
 import org.talend.core.hadoop.repository.HadoopRepositoryUtil;
 import org.talend.core.model.metadata.IMetadataConnection;
@@ -121,13 +125,29 @@ public class HiveClassLoaderFactory {
         }
 
         Object[] configurationJars = new String[0];
+        Consumer<DynamicClassLoader> afterLoad = null;
         String useCustomConfs = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_USE_CUSTOM_CONFS);
         if (Boolean.valueOf(useCustomConfs)) {
             String clusterId = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_HADOOP_CLUSTER_ID);
-            String customConfsJarName = getCustomConfsJarName(clusterId);
-            if (customConfsJarName != null) {
-                configurationJars = new String[] { customConfsJarName };
+
+            HadoopConfJarBean confJarBean = getCustomConfsJarName(clusterId);
+            if (confJarBean != null) {
+                if (confJarBean.isOverrideCustomConf()) {
+                    String overrideCustomConfPath = confJarBean.getOriginalOverrideCustomConfPath();
+                    if (StringUtils.isBlank(overrideCustomConfPath) || !new File(overrideCustomConfPath).exists()) {
+                        ExceptionHandler.process(
+                                new Exception("Set Hadoop configuration JAR path is invalid: " + overrideCustomConfPath));
+                    } else {
+                        afterLoad = (t) -> t.addLibrary(overrideCustomConfPath);
+                    }
+                } else {
+                    String customConfsJarName = confJarBean.getCustomConfJarName();
+                    if (customConfsJarName != null) {
+                        configurationJars = new String[] { customConfsJarName };
+                    }
+                }
             }
+
         } else {
             String useKrb = (String) metadataConn.getParameter(ConnParameterKeys.CONN_PARA_KEY_USE_KRB);
             if (Boolean.valueOf(useKrb)) {
@@ -158,12 +178,15 @@ public class HiveClassLoaderFactory {
                 loader.addLibrary(jarFile.getAbsolutePath());
             }
         }
+        if (afterLoad != null) {
+            afterLoad.accept(loader);
+        }
     }
 
-    private static String getCustomConfsJarName(String clusterId) {
+    private static HadoopConfJarBean getCustomConfsJarName(String clusterId) {
         IHadoopClusterService hadoopClusterService = HadoopRepositoryUtil.getHadoopClusterService();
         if (hadoopClusterService != null) {
-            return hadoopClusterService.getCustomConfsJarName(clusterId);
+            return hadoopClusterService.getCustomConfsJar(clusterId).orElse(null);
         }
         return null;
     }