fix(TUP-29381)CVE on (#4223)

commons-codec-1.6.jar/commons-codec-1.9.jar/commons-codec-1.10.jar

main/features/org.talend.jdt.feature/feature.xml

@@ -29,7 +29,7 @@
     <import plugin="org.eclipse.jdt.launching" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.eclipse.jdt.ui" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.hamcrest.core" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.junit" version="0.0.0" match="greaterOrEqual"/>
+    <import plugin="org.junit" version="4.13.2" match="greaterOrEqual"/>
   </requires>
   <plugin id="org.eclipse.jdt.launching.macosx" os="macosx" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
   <plugin id="org.eclipse.jdt.launching.ui.macosx" os="macosx" download-size="0" install-size="0" version="0.0.0" unpack="false"/>

main/features/org.talend.libraries.jdbc.feature/feature.xml

@@ -6,14 +6,12 @@
   <license url="http://www.example.com/license">[Enter License Description here.]</license>  
   <plugin id="org.talend.libraries.jdbc.access" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.jdbc.as400" download-size="0" install-size="0" version="0.0.0"/>
-  <plugin id="org.talend.libraries.jdbc.derby" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.jdbc.exasol" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.jdbc.h2" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.jdbc.hsql" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.jdbc.ingres" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.jdbc.mysql" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.jdbc.paraccel" download-size="0" install-size="0" version="0.0.0"/>
-  <plugin id="org.talend.libraries.jdbc.postgresql" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.jdbc.sqlite3" download-size="0" install-size="0" version="0.0.0"/>
   <plugin id="org.talend.libraries.jdbc.teradata" download-size="0" install-size="0" version="0.0.0"/>
 </feature>

main/features/org.talend.utils.feature/feature.xml

@@ -12,13 +12,13 @@
     <import plugin="com.thoughtworks.paranamer" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="jackson-core-asl" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="jackson-mapper-asl" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.apache.commons.codec" version="0.0.0" match="greaterOrEqual"/>
+    <import plugin="org.apache.commons.codec" version="1.14.0" match="greaterOrEqual"/>
     <import plugin="org.apache.commons.compress" version="1.10.0" match="greaterOrEqual"/>
     <import plugin="org.apache.commons.lang" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.apache.commons.lang3" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.apache.log4j" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.apache.servicemix.bundles.avro" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.junit" version="0.0.0" match="greaterOrEqual"/>
+    <import plugin="org.junit" version="4.13.2" match="greaterOrEqual"/>
     <import plugin="org.slf4j.api" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.apache.commons.configuration" version="2.0.0" match="greaterOrEqual"/>
   </requires>  

main/plugins/org.talend.common.ui.runtime/src/main/java/org/talend/commons/ui/runtime/image/ECoreImage.java

@@ -77,6 +77,7 @@ public enum ECoreImage implements IImage {
     ROUTES_ICON("/icons1/routes_icon.png"), //$NON-NLS-1$
     ROUTINE_ICON("/icons1/routine.png"), //$NON-NLS-1$
     ROUTINE_EDITOR_ICON("/icons1/routine_editor.png"), //$NON-NLS-1$
+    ROUTINESJAR_ICON("/icons1/routinesjar.png"), //$NON-NLS-1$
     JOB_SCRIPTS_ICON("/icons1/job_scripts.png"), //$NON-NLS-1$
     JOB_SCRIPTS_TOOLBAR_ICON("/icons1/job_script_toolbar.png"), //$NON-NLS-1$
     FTP_ICON("/icons1/ftp.png"), //$NON-NLS-1$
@@ -166,6 +167,7 @@ public enum ECoreImage implements IImage {
     RECYCLE_BIN_OVERLAY("/icons1/recycle_bin_overlay.gif"), //$NON-NLS-1$
     DELETED_OVERLAY("/icons1/deleted_overlay.gif"), //$NON-NLS-1$
     IMPORT_JAR("/icons1/importjar.gif"), //$NON-NLS-1$
+    SHARE_LIBS("/icons1/share.png"), //$NON-NLS-1$
     REFERENCED_ICON("/icons1/referenced.png"), //$NON-NLS-1$
 
     CDC_ADDED_OVERLAY("/icons1/cdc_added_overlay.png"), //$NON-NLS-1$

main/plugins/org.talend.common.ui.runtime/src/main/java/org/talend/commons/ui/runtime/ws/WindowSystem.java

@@ -16,6 +16,8 @@ import org.eclipse.core.internal.runtime.InternalPlatform;
 import org.eclipse.core.runtime.Platform;
 import org.osgi.framework.BundleContext;
 
+
+
 /**
  *
  * DOC root class global comment. Detailled comment <br/>
@@ -52,5 +54,9 @@ public class WindowSystem {
     public static boolean isOSX() {
         return Platform.WS_CARBON.equals(ws) || Platform.WS_COCOA.equals(ws);
     }
+    
+    public static boolean isBigSurOrLater () {
+    	return isOSX() && ("10.16".compareTo(System.getProperty("os.version","0")) <= 0);
+    }
 
 }

main/plugins/org.talend.commons.runtime/META-INF/MANIFEST.MF

@@ -16,7 +16,8 @@ Require-Bundle: org.apache.log4j;visibility:=reexport,
  org.talend.utils,
  org.eclipse.core.net,
  org.eclipse.m2e.core,
- org.eclipse.m2e.maven.runtime
+ org.eclipse.m2e.maven.runtime,
+ org.eclipse.core.resources
 Export-Package: org.talend.commons,
  org.talend.commons.exception,
  org.talend.commons.i18n,

main/plugins/org.talend.commons.runtime/pom_server.xml

@@ -92,7 +92,7 @@
         <dependency>
             <groupId>xerces</groupId>
             <artifactId>xercesImpl</artifactId>
-            <version>2.11.0</version>
+            <version>2.12.0</version>
         </dependency>
         <dependency>
             <groupId>log4j</groupId>

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/VersionUtils.java

@@ -232,10 +232,50 @@ public class VersionUtils {
      * Check if studio version < other studio version record in remote project.
      */
     public static boolean isInvalidProductVersion(String remoteFullProductVersion) {
+        String localProductVersion = getInternalVersion();
+        return isInvalidProductVersion(localProductVersion, remoteFullProductVersion);
+    }
+
+    protected static boolean isInvalidProductVersion(String localProductVersion, String remoteFullProductVersion) {
         if (remoteFullProductVersion == null) {
             return false;
         }
-        return getInternalVersion().compareTo(getProductVersionWithoutBranding(remoteFullProductVersion)) < 0;
+        if (skipCheckingNightlyBuilds(localProductVersion, remoteFullProductVersion)) {
+            return false;
+        }
+        return localProductVersion.compareTo(getProductVersionWithoutBranding(remoteFullProductVersion)) < 0;
+    }
+
+    public static boolean productVersionIsNewer(String remoteFullProductVersion) {
+        String localProductVersion = getInternalVersion();
+        return productVersionIsNewer(localProductVersion, remoteFullProductVersion);
+    }
+
+    protected static boolean productVersionIsNewer(String localProductVersion, String remoteFullProductVersion) {
+        if (remoteFullProductVersion == null) {
+            return false;
+        }
+        if (skipCheckingNightlyBuilds(localProductVersion, remoteFullProductVersion)) {
+            return false;
+        }
+        return localProductVersion.compareTo(getProductVersionWithoutBranding(remoteFullProductVersion)) > 0;
+    }
+
+    private static boolean skipCheckingNightlyBuilds(String localProductVersion, String remoteFullProductVersion) {
+        String separator = "-"; //$NON-NLS-1$
+        String localSuffix = StringUtils.substringAfterLast(localProductVersion, separator);
+
+        String remoteProductVersion = getProductVersionWithoutBranding(remoteFullProductVersion);
+        String remoteSuffix = StringUtils.substringAfterLast(remoteProductVersion, separator);
+
+        String nightly = "SNAPSHOT"; //$NON-NLS-1$
+        String milestone = "M"; //$NON-NLS-1$
+        if ((localSuffix.equals(nightly) || localSuffix.startsWith(milestone))
+                && (remoteSuffix.equals(nightly) || remoteSuffix.startsWith(milestone))) {
+            // skip checking between nightly/milestone build.
+            return true;
+        }
+        return false;
     }
 
     public static String getTalendVersion(String productVersion) {
@@ -310,4 +350,24 @@ public class VersionUtils {
         }
     }
 
+    public static String getSimplifiedPatchName(String projectPatchName) {
+
+        if (projectPatchName != null) {
+            String result = null;
+            if (projectPatchName.contains("_") && projectPatchName.split("_").length >= 3) {
+                result = projectPatchName.split("_")[2];
+                if (!result.startsWith("R")) {
+                    return null;
+                }
+            }
+            if (projectPatchName.contains("-")) {
+                String[] split = projectPatchName.split("-");
+                if (split != null && split.length > 0) {
+                    return result + "-" + split[split.length - 1];
+                }
+            }
+        }
+        return null;
+    }
+
 }

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/database/SAPHanaDataBaseMetadata.java

@@ -148,7 +148,7 @@ public class SAPHanaDataBaseMetadata extends FakeDatabaseMetaData {
         }
 
         // For Calculation View
-        if (ArrayUtils.contains(neededTypes, NEEDED_TYPES[3])) {
+        if (ArrayUtils.contains(neededTypes, NEEDED_TYPES[3]) && "_SYS_BIC".equalsIgnoreCase(schemaPattern)) { //$NON-NLS-1$
             // check if the type is contained is in the types needed.
             String sqlcv = "SELECT OBJECT_NAME,PACKAGE_ID FROM _SYS_REPO.ACTIVE_OBJECT WHERE OBJECT_SUFFIX = 'calculationview'"; //$NON-NLS-1$
             if (tableNamePattern != null && !tableNamePattern.equals("%")) { //$NON-NLS-1$
@@ -171,7 +171,7 @@ public class SAPHanaDataBaseMetadata extends FakeDatabaseMetaData {
                         packageId = packageId.trim();
                     }
                     String name = packageId + "/" + objectName; //$NON-NLS-1$
-                    String[] r = new String[] { "", "_SYS_BIC", name, NEEDED_TYPES[3], "", packageId }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+                    String[] r = new String[] { "", schemaPattern, name, NEEDED_TYPES[3], "", packageId }; //$NON-NLS-1$ //$NON-NLS-2$
                     listcv.add(r);
                 }
             } catch (SQLException e) {
@@ -355,4 +355,16 @@ public class SAPHanaDataBaseMetadata extends FakeDatabaseMetaData {
         tableResultSet.setData(list);
         return tableResultSet;
     }
+
+    @Override
+    public String getDatabaseProductName() throws SQLException {
+        return this.connection.getMetaData().getDatabaseProductName();
+
+    }
+
+    @Override
+    public String getDatabaseProductVersion() throws SQLException {
+        return this.connection.getMetaData().getDatabaseProductVersion();
+    }
+
 }

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/network/NetworkUtil.java

@@ -15,6 +15,7 @@ package org.talend.commons.utils.network;
 import java.lang.reflect.Field;
 import java.net.Authenticator;
 import java.net.HttpURLConnection;
+import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.NetworkInterface;
 import java.net.PasswordAuthentication;
@@ -22,7 +23,11 @@ import java.net.SocketException;
 import java.net.URI;
 import java.net.URL;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
 import java.util.Enumeration;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 import org.apache.commons.lang.StringUtils;
@@ -150,6 +155,28 @@ public class NetworkUtil {
 
                 @Override
                 protected PasswordAuthentication getPasswordAuthentication() {
+                    String httpProxyHost = System.getProperty("http.proxyHost"); //$NON-NLS-1$
+                    String httpProxyPort = System.getProperty("http.proxyPort"); //$NON-NLS-1$
+                    String httpsProxyHost = System.getProperty("https.proxyHost"); //$NON-NLS-1$
+                    String httpsProxyPort = System.getProperty("https.proxyPort"); //$NON-NLS-1$
+                    String requestingHost = getRequestingHost();
+                    int requestingPort = getRequestingPort();
+                    String proxyHost = null;
+                    String proxyPort = null;
+                    boolean isHttp = false;
+                    if ("http".equalsIgnoreCase(getRequestingScheme())) {
+                        isHttp = true;
+                    }
+                    if (isHttp && StringUtils.isNotBlank(httpProxyHost)) {
+                        proxyHost = httpProxyHost;
+                        proxyPort = httpProxyPort;
+                    } else {
+                        proxyHost = httpsProxyHost;
+                        proxyPort = httpsProxyPort;
+                    }
+                    if (!StringUtils.equals(proxyHost, requestingHost) || !StringUtils.equals(proxyPort, "" + requestingPort)) {
+                        return null;
+                    }
                     String httpProxyUser = System.getProperty("http.proxyUser"); //$NON-NLS-1$
                     String httpProxyPassword = System.getProperty("http.proxyPassword"); //$NON-NLS-1$
                     String httpsProxyUser = System.getProperty("https.proxyUser"); //$NON-NLS-1$
@@ -167,7 +194,11 @@ public class NetworkUtil {
                             proxyPassword = httpsProxyPassword.toCharArray();
                         }
                     }
-                    return new PasswordAuthentication(proxyUser, proxyPassword);
+                    if (StringUtils.isBlank(proxyUser)) {
+                        return null;
+                    } else {
+                        return new PasswordAuthentication(proxyUser, proxyPassword);
+                    }
                 }
 
             });
@@ -206,6 +237,54 @@ public class NetworkUtil {
         }
     }
 
+    public static List<String> getLocalLoopbackAddresses(boolean wrapIpV6) {
+        Set<String> addresses = new LinkedHashSet<>();
+        try {
+            addresses.add(getIp(InetAddress.getLoopbackAddress(), wrapIpV6));
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+
+        try {
+            Enumeration<NetworkInterface> networkInterfaces = NetworkInterface.getNetworkInterfaces();
+            while (networkInterfaces.hasMoreElements()) {
+                NetworkInterface networkInterface = networkInterfaces.nextElement();
+                Enumeration<InetAddress> inetAddresses = networkInterface.getInetAddresses();
+                while (inetAddresses.hasMoreElements()) {
+                    InetAddress inetAddress = inetAddresses.nextElement();
+                    if (inetAddress != null && inetAddress.isLoopbackAddress()) {
+                        addresses.add(getIp(inetAddress, wrapIpV6));
+                    }
+                }
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+
+        if (addresses.isEmpty()) {
+            addresses.add("127.0.0.1");
+            String ipv6Loopback = "::1";
+            if (wrapIpV6) {
+                ipv6Loopback = "[" + ipv6Loopback + "]";
+            }
+            addresses.add(ipv6Loopback);
+        }
+
+        return new ArrayList<>(addresses);
+    }
+
+    private static String getIp(InetAddress inetAddress, boolean wrapIpV6) {
+        if (wrapIpV6 && Inet6Address.class.isInstance(inetAddress)) {
+            String addr = inetAddress.getHostAddress();
+            if (!addr.startsWith("[") || !addr.endsWith("]")) {
+                addr = "[" + addr + "]";
+            }
+            return addr;
+        } else {
+            return inetAddress.getHostAddress();
+        }
+    }
+
     public static boolean isSelfAddress(String addr) {
         if (addr == null || addr.isEmpty()) {
             return false; // ?

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/network/TalendProxySelector.java

@@ -25,6 +25,7 @@ import java.net.URISyntaxException;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -38,6 +39,8 @@ import java.util.Set;
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Priority;
 import org.eclipse.core.internal.net.ProxyManager;
+import org.eclipse.core.internal.net.ProxyType;
+import org.eclipse.core.net.proxy.IProxyChangeEvent;
 import org.eclipse.core.net.proxy.IProxyService;
 import org.talend.commons.CommonsPlugin;
 import org.talend.commons.exception.ExceptionHandler;
@@ -77,6 +80,14 @@ public class TalendProxySelector extends ProxySelector {
 
     private static final String PROP_DISABLE_DEFAULT_SELECTOR_PROVIDER = "talend.studio.proxy.disableDefaultSelectorProvider";
 
+    private static final String PROP_PROXY_EXCLUDE_LOOPBACK_ADDRESS_AUTOMATICALLY = "talend.studio.proxy.excludeLoopbackAutomatically";
+
+    private static final String PROP_PROXY_EXCLUDE_LOOPBACK_ADDRESS_AUTOMATICALLY_DEFAULT = "true";
+
+    private static final String PROP_PROXY_HTTP_NON_PROXYHOSTS = "http.nonProxyHosts";
+
+    private static final String PROP_PROXY_HTTPS_NON_PROXYHOSTS = "https.nonProxyHosts";
+
     /**
      * Example: update.talend.com,socket:http,https:http;nexus.talend.com,socket,http;,socket:http
      */
@@ -104,6 +115,8 @@ public class TalendProxySelector extends ProxySelector {
 
     private EProxySelector eProxySelector;
 
+    private IProxyService proxyManager;
+
     final private Map<Object, Collection<IProxySelectorProvider>> selectorProviders;
 
     private Map<String, Map<String, String>> hostMap;
@@ -114,6 +127,10 @@ public class TalendProxySelector extends ProxySelector {
 
     private static Object instanceLock = new Object();
 
+    private List<String> localLoopbackAddresses;
+
+    private Object localLoopbackAddressesLock = new Object();
+
     private boolean printProxyLog = false;
 
     private boolean allowProxyRedirect = false;
@@ -126,6 +143,8 @@ public class TalendProxySelector extends ProxySelector {
 
     private boolean updateSystemPropertiesForJre = true;
 
+    private boolean excludeLoopbackAddressAutomatically = false;
+
     private TalendProxySelector(final ProxySelector eclipseDefaultSelector) {
         this.eclipseDefaultSelector = eclipseDefaultSelector;
         this.jreDefaultSelector = new DefaultProxySelector();
@@ -139,6 +158,8 @@ public class TalendProxySelector extends ProxySelector {
         executeConnectionFailed = Boolean.valueOf(System.getProperty(PROP_EXECUTE_CONNECTION_FAILED, Boolean.TRUE.toString()));
         updateSystemPropertiesForJre = Boolean
                 .valueOf(System.getProperty(PROP_UPDATE_SYSTEM_PROPERTIES_FOR_JRE, Boolean.TRUE.toString()));
+        excludeLoopbackAddressAutomatically = Boolean.valueOf(System.getProperty(
+                PROP_PROXY_EXCLUDE_LOOPBACK_ADDRESS_AUTOMATICALLY, PROP_PROXY_EXCLUDE_LOOPBACK_ADDRESS_AUTOMATICALLY_DEFAULT));
 
         switch (System.getProperty(PROP_PROXY_SELECTOR, PROP_PROXY_SELECTOR_DEFAULT).toLowerCase()) {
         case PROP_PROXY_SELECTOR_JRE:
@@ -148,11 +169,83 @@ public class TalendProxySelector extends ProxySelector {
             this.eProxySelector = EProxySelector.eclipse_default;
             break;
         }
+        proxyManager = ProxyManager.getProxyManager();
+        checkProxyManager(IProxyChangeEvent.PROXY_DATA_CHANGED);
+        proxyManager.addProxyChangeListener(event -> checkProxyManager(event.getChangeType()));
 
         initHostMap();
         initRedirectList();
     }
 
+    private void checkProxyManager(int changeEvent) {
+        try {
+            if (IProxyChangeEvent.PROXY_DATA_CHANGED == changeEvent
+                    || IProxyChangeEvent.NONPROXIED_HOSTS_CHANGED == changeEvent) {
+                if (this.excludeLoopbackAddressAutomatically && proxyManager.isProxiesEnabled()) {
+                    List<String> addresses = getLocalLoopbackAddresses();
+                    if (addresses != null && !addresses.isEmpty()) {
+                        if (org.eclipse.core.internal.net.ProxySelector
+                                .canSetBypassHosts(org.eclipse.core.internal.net.ProxySelector.getDefaultProvider())) {
+                            List<String> configuredProxies = Arrays.asList(proxyManager.getNonProxiedHosts());
+                            if (!configuredProxies.containsAll(addresses)) {
+                                Set<String> nonProxyHosts = new HashSet<>(addresses);
+                                nonProxyHosts.addAll(configuredProxies);
+                                ExceptionHandler.log(
+                                        this.getClass().getName() + ":" + "-D" + PROP_PROXY_EXCLUDE_LOOPBACK_ADDRESS_AUTOMATICALLY
+                                                + "=true, adding missing loopback addresses into eclipse nonProxyHosts: "
+                                                + nonProxyHosts);
+                                proxyManager.setNonProxiedHosts(nonProxyHosts.toArray(new String[0]));
+                            }
+                        } else {
+                            updateNonProxyHosts(addresses, PROP_PROXY_HTTP_NON_PROXYHOSTS);
+                            updateNonProxyHosts(addresses, PROP_PROXY_HTTPS_NON_PROXYHOSTS);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
+    private void updateNonProxyHosts(List<String> localLoopbackAddresses, final String nonProxyProperty) {
+        if (localLoopbackAddresses != null && !localLoopbackAddresses.isEmpty()) {
+            Set<String> nonProxyHosts = new HashSet<>(localLoopbackAddresses);
+            String property = System.getProperty(nonProxyProperty);
+            boolean update = true;
+            if (StringUtils.isNotBlank(property)) {
+                List<String> configuredProxies = Arrays.asList(ProxyType.convertPropertyStringToHosts(property));
+                if (configuredProxies.containsAll(localLoopbackAddresses)) {
+                    update = false;
+                } else {
+                    nonProxyHosts.addAll(configuredProxies);
+                }
+            }
+            if (update) {
+                ExceptionHandler.log(this.getClass().getName() + ":" + "-D" + PROP_PROXY_EXCLUDE_LOOPBACK_ADDRESS_AUTOMATICALLY
+                        + "=true, adding missing loopback addresses into " + nonProxyProperty + ": " + nonProxyHosts);
+                System.setProperty(nonProxyProperty,
+                        ProxyType.convertHostsToPropertyString(nonProxyHosts.toArray(new String[0])));
+            }
+        }
+    }
+
+    private List<String> getLocalLoopbackAddresses() {
+        if (this.localLoopbackAddresses == null) {
+            synchronized (localLoopbackAddressesLock) {
+                if (this.localLoopbackAddresses == null) {
+                    List<String> addresses = NetworkUtil.getLocalLoopbackAddresses(false);
+                    final String localhost = "localhost";
+                    if (!addresses.contains(localhost)) {
+                        addresses.add(localhost);
+                    }
+                    this.localLoopbackAddresses = addresses;
+                }
+            }
+        }
+        return this.localLoopbackAddresses;
+    }
+
     private void initHostMap() {
         try {
             hostMap = new HashMap<>();
@@ -282,7 +375,27 @@ public class TalendProxySelector extends ProxySelector {
             ExceptionHandler.log("TalendProxySelector.select " + uri);
         }
         if (uri == null) {
-            return Collections.EMPTY_LIST;
+            List<Proxy> result = new ArrayList<>();
+            result.add(Proxy.NO_PROXY);
+            return result;
+        }
+        try {
+            if (this.excludeLoopbackAddressAutomatically) {
+                List<String> addresses = getLocalLoopbackAddresses();
+                if (addresses != null) {
+                    String host = uri.getHost();
+                    if (addresses.contains(host)) {
+                        if (printProxyLog) {
+                            ExceptionHandler.log(uri + " is excluded from proxy");
+                        }
+                        List<Proxy> result = new ArrayList<>();
+                        result.add(Proxy.NO_PROXY);
+                        return result;
+                    }
+                }
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
         }
         URI validatedUri = validateUri(uri);
         Set<Proxy> results = new LinkedHashSet<>();
@@ -332,6 +445,9 @@ public class TalendProxySelector extends ProxySelector {
             ExceptionHandler.log("Selected proxys for " + uri + ", " + proxys);
             ExceptionHandler.process(new Exception("Proxy call stacks"), Priority.INFO);
         }
+        if (results.isEmpty()) {
+            results.add(Proxy.NO_PROXY);
+        }
         return new LinkedList<Proxy>(results);
     }
 

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/system/EclipseCommandLine.java

@@ -49,11 +49,17 @@ public class EclipseCommandLine {
      */
     static public final String TALEND_PROJECT_TYPE_COMMAND = "-talendProjectType"; //$NON-NLS-1$
 
+    static public final String TALEND_LICENCE_PATH = "talend.licence.path"; //$NON-NLS-1$
+
+    static public final String ARG_TALEND_LICENCE_PATH = "-" + TALEND_LICENCE_PATH; //$NON-NLS-1$
+
     /**
      * for relaunch of the plugins when relaunching the Studio
      */
     static public final String TALEND_RELOAD_COMMAND = "-talendReload"; //$NON-NLS-1$
 
+    static public final String LOGIN_ONLINE_UPDATE = "--loginOnlineUpdate";
+
     static public final String ARG_TALEND_BUNDLES_CLEANED = "-talend.studio.bundles.cleaned"; //$NON-NLS-1$
 
     static public final String PROP_TALEND_BUNDLES_DO_CLEAN = "-talend.studio.bundles.doclean"; //$NON-NLS-1$

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/time/PerformanceLogManager.java

@@ -0,0 +1,127 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.commons.utils.time;
+
+import java.util.Enumeration;
+import java.util.Properties;
+
+import org.apache.log4j.Hierarchy;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PropertyConfigurator;
+import org.apache.log4j.RollingFileAppender;
+import org.apache.log4j.spi.LoggerFactory;
+import org.apache.log4j.spi.RootLogger;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.Platform;
+
+public class PerformanceLogManager {
+
+	private Hierarchy hierarchy;
+	
+	public PerformanceLogManager() {
+	    Properties properties = new Properties();
+	    properties.put("log4j.rootCategory", ", A1");
+	    properties.put("log4j.appender.A1", RollingFileAppender.class.getName());
+	    IPath performanceLogPath = Platform.getLogFileLocation().removeLastSegments(1).append("performance.log");
+	    properties.put("log4j.appender.A1.File", performanceLogPath.toOSString());
+	    properties.put("log4j.appender.A1.MaxBackupIndex", "10");// same as .log's max backup log file count
+	    properties.put("log4j.appender.A1.MaxFileSize", "1000000");//1000*1000 byte, same as .log's max file size
+	    properties.put("log4j.appender.A1.layout", "org.apache.log4j.PatternLayout");
+	    properties.put("log4j.appender.A1.layout.ConversionPattern", "%d %-5p %c %x - %m%n");
+	    
+		this.hierarchy = new Hierarchy(new RootLogger(Level.INFO));
+		new PropertyConfigurator().doConfigure(properties,hierarchy);	
+	}
+	
+	/**
+	 * Checks if this PluginLogManager is disabled for this level.
+	 * @param level level value
+	 * @return boolean true if it is disabled
+	 */
+	public boolean isDisabled(int level) {
+		return this.hierarchy.isDisabled(level);
+	}
+	
+	/**
+	 * Enable logging for logging requests with level l or higher.
+	 * By default all levels are enabled.
+	 * @param level level object
+	 */
+	public void setThreshold(Level level) {
+		this.hierarchy.setThreshold(level);
+	}
+	
+	/**
+	 * The string version of setThreshold(Level level)
+	 * @param level level string
+	 */
+	public void setThreshold(String level) {
+		this.hierarchy.setThreshold(level);
+	}
+
+	/**
+	 * Get the repository-wide threshold.
+	 * @return Level
+	 */
+	public Level getThreshold() {
+		return this.hierarchy.getThreshold();
+	}
+
+	/**
+	 * Returns a new logger instance named as the first parameter
+	 * using the default factory. If a logger of that name already exists,
+	 * then it will be returned. Otherwise, a new logger will be instantiated 
+	 * and then linked with its existing ancestors as well as children.
+	 * @param name logger name
+	 * @return Logger
+	 */
+	public Logger getLogger(String name) {
+		return this.hierarchy.getLogger(name);
+	}
+	
+	/**
+	 * The same as getLogger(String name) but using a factory instance instead of
+	 * a default factory.
+	 * @param name logger name
+	 * @param factory factory instance 
+	 * @return Logger
+	 */
+	public Logger getLogger(String name, LoggerFactory factory) {
+		return this.hierarchy.getLogger(name,factory);
+	}
+
+	public Logger getRootLogger() {
+		return this.hierarchy.getRootLogger();
+	}
+
+	public Logger exists(String name) {
+		return this.hierarchy.exists(name);
+	}
+	
+	public void shutdown() {
+	    this.hierarchy.shutdown();
+	}
+	
+	/**
+	 * Returns all the loggers in this manager.
+	 * @return Enumeration logger enumeration
+	 */
+	public Enumeration getCurrentLoggers() {
+		return this.hierarchy.getCurrentLoggers();
+	}
+
+	public void resetConfiguration() {
+		this.hierarchy.resetConfiguration();
+	}
+}

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/time/PerformanceStatisticUtil.java

@@ -0,0 +1,337 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.commons.utils.time;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.eclipse.core.resources.IWorkspaceRoot;
+import org.eclipse.core.resources.ResourcesPlugin;
+import org.eclipse.core.runtime.Platform;
+import org.talend.commons.exception.CommonExceptionHandler;
+
+/**
+ * DOC sbliu class global comment. Detailled comment
+ */
+public class PerformanceStatisticUtil {
+
+    private static final int MEGABYTE = 1024 * 1024;// MB = 1024*1024 byte
+
+    private static final int KILOBYTE = 1024;// kb=1024 byte
+
+    private static final int numOfBlocks = 256;
+
+    private static final int blockSizeKb = 512;
+
+    private static final String dataFile = "testio.data";
+
+    private static String recordingFileName = "performance_record";
+
+    private static File recordingFile = null;
+
+    private static enum BlockSequence {
+        SEQUENTIAL,
+        RANDOM;
+    }
+
+    public static enum StatisticKeys {
+
+        IO_COUNT("I/O.count"), // io count
+        IO_W_MB_SEC("I/O.write"), // write speed MB
+        IO_R_MB_SEC("I/O.read"), // read speed MB
+        IO_W_AVERAGE_MB_SEC("I/O.write.average"), // average speed of write MB
+        IO_R_AVERAGE_MB_SEC("I/O.read.average"), // average speed of read
+
+        STARTUP_AVERAGE("startup.average"),
+        STARTUP_MAX("startup.max"),
+        STARTUP_COUNT("startup.count");
+
+        private String key;
+
+        StatisticKeys(String _key) {
+            key = _key;
+        }
+
+        public String get() {
+            return key;
+        }
+    }
+
+    public static void recordStartupEpapsedTime(double elapsedTimeInSeconds) {
+        File file = getRecordingFile();
+
+        Properties props = PropertiesFileUtil.read(file, true);
+        String propCount = props.getProperty(StatisticKeys.STARTUP_COUNT.get(), "0");
+        String propMax = props.getProperty(StatisticKeys.STARTUP_MAX.get(), "0");
+        String propAverage = props.getProperty(StatisticKeys.STARTUP_AVERAGE.get(), "0");
+
+        int iPropCount = Integer.parseInt(propCount);
+        double iPropMax = Double.parseDouble(propMax);
+        double iPropAverage = Double.parseDouble(propAverage);
+
+        iPropMax = iPropMax > elapsedTimeInSeconds ? iPropMax : elapsedTimeInSeconds;
+        iPropAverage = (iPropAverage * iPropCount + elapsedTimeInSeconds) / (iPropCount + 1);
+        iPropCount++;
+
+        props.setProperty(StatisticKeys.STARTUP_COUNT.get(), "" + iPropCount);
+        props.setProperty(StatisticKeys.STARTUP_MAX.get(), "" + iPropMax);
+        props.setProperty(StatisticKeys.STARTUP_AVERAGE.get(), "" + iPropAverage);
+
+        PropertiesFileUtil.store(file, props);
+    }
+
+    public static File getRecordingFile() {
+        if (recordingFile != null) {
+            return recordingFile;
+        }
+
+        String configurationLocation = Platform.getConfigurationLocation().getURL().getPath();
+        File file = new File(configurationLocation + "/data_collector/" + recordingFileName);
+        File oldFile = new File(configurationLocation + "/" + recordingFileName);
+        if(oldFile.exists()) {
+            if(!file.exists()) {
+                if(!file.getParentFile().exists()) {
+                    file.getParentFile().mkdirs();
+                }
+                try {
+                    Files.move(Paths.get(oldFile.toURI()), Paths.get(file.toURI()), StandardCopyOption.ATOMIC_MOVE);
+                } catch (IOException e) {
+                    CommonExceptionHandler.log(e.getMessage());
+                }
+            }
+            
+            try {
+                Files.deleteIfExists(Paths.get(oldFile.toURI()));
+            } catch (IOException e) {
+                CommonExceptionHandler.log(e.getMessage());
+            }
+        }
+        
+        return file;
+    }
+
+    public static void setRecordingFile(File _recordingFile) {
+        recordingFile = _recordingFile;
+    }
+    
+    public static void reset() {
+        File _recordingFile = getRecordingFile();
+        try {
+            Files.deleteIfExists(Paths.get(_recordingFile.toURI()));
+        } catch (IOException e) {
+            CommonExceptionHandler.log(e.getMessage());
+        }
+    }
+
+    private static Lock lock = new ReentrantLock();
+    private static Condition condition = lock.newCondition();
+    private static boolean measureIOFinished = true;
+    
+    public static void waitUntilFinish() throws InterruptedException {
+        lock.lock();
+
+        try {
+            if(!measureIOFinished) {
+                condition.await(20, TimeUnit.SECONDS);
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+    
+    public static void measureIO() {
+        new Thread()  {
+            public void run() {
+                measureIOFinished = false;
+                try {
+                    _measureIO();
+                } finally {
+                    measureIOFinished = true;
+                }
+            }
+        }.start();
+    }
+
+    private static void _measureIO() {
+        File file = getRecordingFile();
+        Properties props = PropertiesFileUtil.read(file, true);
+
+        IWorkspaceRoot root = ResourcesPlugin.getWorkspace().getRoot();
+        File workspace = root.getLocation().makeAbsolute().toFile();
+        File locationDir = new File(workspace, "temp"); // here is workspace/temp dir
+        File testFile = detectTestDataFile(locationDir);
+
+        if (testFile != null) {
+            measureWrite(props, testFile);
+            measureRead(props, testFile);
+
+            PropertiesFileUtil.store(file, props);
+        }
+    }
+    
+    private static void measureWrite(Properties props, File testFile) {
+        int blockSize = blockSizeKb * KILOBYTE;
+
+        long startTime = System.nanoTime();
+        long totalBytesWrittenInMark = writeIO(numOfBlocks, BlockSequence.RANDOM, blockSize, testFile);
+        totalBytesWrittenInMark = totalBytesWrittenInMark + writeIO(numOfBlocks, BlockSequence.SEQUENTIAL, blockSize, testFile);
+        long endTime = System.nanoTime();
+
+        long elapsedTimeNs = endTime - startTime;
+        double sec = (double) elapsedTimeNs / (double) 1000000000;
+        double mbWritten = (double) totalBytesWrittenInMark / (double) MEGABYTE;
+        double bwMbSec = mbWritten / sec;
+
+        String ioCount = props.getProperty(StatisticKeys.IO_COUNT.get(), "0");
+        String ioWAverageMbSec = props.getProperty(StatisticKeys.IO_W_AVERAGE_MB_SEC.get(), "0");
+        String ioWMbSec = props.getProperty(StatisticKeys.IO_W_MB_SEC.get(), "0");
+
+        int digital_ioCount = Integer.parseInt(ioCount);
+        double digital_ioWAverageMbSec = Double.parseDouble(ioWAverageMbSec);
+        double digital_ioWMbSec = Double.parseDouble(ioWMbSec);
+
+        digital_ioWAverageMbSec = (digital_ioWAverageMbSec * digital_ioCount + bwMbSec) / (digital_ioCount + 1);
+        digital_ioWMbSec = bwMbSec;
+
+        props.setProperty(StatisticKeys.IO_W_AVERAGE_MB_SEC.get(), format(digital_ioWAverageMbSec));
+        props.setProperty(StatisticKeys.IO_W_MB_SEC.get(), format(digital_ioWMbSec));
+    }
+
+    private static long writeIO(int numOfBlocks, BlockSequence blockSequence, int blockSize, File testFile) {
+        byte[] blockArr = new byte[blockSize];
+        for (int b = 0; b < blockArr.length; b++) {
+            if (b % 2 == 0) {
+                blockArr[b] = (byte) 0xFF;
+            }
+        }
+        String mode = "rwd";// "rwd"
+
+        long totalBytesWrittenInMark = 0;
+        try {
+            try (RandomAccessFile rAccFile = new RandomAccessFile(testFile, mode)) {
+                for (int b = 0; b < numOfBlocks; b++) {
+                    if (blockSequence == BlockSequence.RANDOM) {
+                        int rLoc = randInt(0, numOfBlocks - 1);
+                        rAccFile.seek(rLoc * blockSize);
+                    } else {
+                        rAccFile.seek(b * blockSize);
+                    }
+                    rAccFile.write(blockArr, 0, blockSize);
+                    totalBytesWrittenInMark += blockSize;
+                }
+            }
+        } catch (IOException e) {
+            CommonExceptionHandler.log(e.getMessage());
+        }
+
+        return totalBytesWrittenInMark;
+    }
+
+    private static File detectTestDataFile(File location) {
+        if (!location.exists()) {
+            location.mkdirs();
+        }
+
+        File testFile = null;
+        try {
+            testFile = new File(location.getAbsolutePath() + File.separator + dataFile);
+            testFile.deleteOnExit();
+            testFile.createNewFile();
+        } catch (IOException e) {
+            CommonExceptionHandler.log(e.getMessage());
+        }
+
+        return testFile;
+    }
+
+    public static void measureRead(Properties props, File testFile) {
+        int blockSize = blockSizeKb * KILOBYTE;
+
+        long startTime = System.nanoTime();
+        long totalBytesReadInMark = readIO(numOfBlocks, BlockSequence.RANDOM, blockSize, testFile);
+        totalBytesReadInMark = totalBytesReadInMark + readIO(numOfBlocks, BlockSequence.SEQUENTIAL, blockSize, testFile);
+        long endTime = System.nanoTime();
+        long elapsedTimeNs = endTime - startTime;
+        double sec = (double) elapsedTimeNs / (double) 1000000000;
+        double mbRead = (double) totalBytesReadInMark / (double) MEGABYTE;
+        double bwMbSec = mbRead / sec;
+
+        String ioCount = props.getProperty(StatisticKeys.IO_COUNT.get(), "0");
+        String ioRAverageMbSec = props.getProperty(StatisticKeys.IO_R_AVERAGE_MB_SEC.get(), "0");
+        String ioRMbSec = props.getProperty(StatisticKeys.IO_R_MB_SEC.get(), "0");
+
+        int digital_ioCount = Integer.parseInt(ioCount);
+        double digital_ioRAverageMbSec = Double.parseDouble(ioRAverageMbSec);
+        double digital_ioRMbSec = Double.parseDouble(ioRMbSec);
+        digital_ioRAverageMbSec = (digital_ioRAverageMbSec * digital_ioCount + bwMbSec) / (digital_ioCount + 1);
+        digital_ioRMbSec = bwMbSec;
+        digital_ioCount++;
+
+        props.setProperty(StatisticKeys.IO_R_AVERAGE_MB_SEC.get(), format(digital_ioRAverageMbSec));
+        props.setProperty(StatisticKeys.IO_R_MB_SEC.get(), format(digital_ioRMbSec));
+        props.setProperty(StatisticKeys.IO_COUNT.get(), "" + digital_ioCount);
+    }
+
+    public static String format(double dvalue) {
+        return BigDecimal.valueOf(dvalue).setScale(2, RoundingMode.HALF_UP).toString();
+    }
+    
+    private static long readIO(int numOfBlocks, BlockSequence blockSequence, int blockSize, File testFile) {
+        long totalBytesReadInMark = 0;
+
+        byte[] blockArr = new byte[blockSize];
+        for (int b = 0; b < blockArr.length; b++) {
+            if (b % 2 == 0) {
+                blockArr[b] = (byte) 0xFF;
+            }
+        }
+        try {
+            try (RandomAccessFile rAccFile = new RandomAccessFile(testFile, "r")) {
+                for (int b = 0; b < numOfBlocks; b++) {
+                    if (blockSequence == BlockSequence.RANDOM) {
+                        int rLoc = randInt(0, numOfBlocks - 1);
+                        rAccFile.seek(rLoc * blockSize);
+                    } else {
+                        rAccFile.seek(b * blockSize);
+                    }
+                    rAccFile.readFully(blockArr, 0, blockSize);
+                    totalBytesReadInMark += blockSize;
+                }
+            }
+        } catch (IOException e) {
+            CommonExceptionHandler.log(e.getMessage());
+        }
+        return totalBytesReadInMark;
+    }
+
+    private static int randInt(int min, int max) {
+        // nextInt is normally exclusive of the top value,
+        // so add 1 to make it inclusive
+        int randomNum = new Random().nextInt((max - min) + 1) + min;
+
+        return randomNum;
+    }
+
+}

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/time/PropertiesCollectorUtil.java

@@ -0,0 +1,55 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.commons.utils.time;
+
+import java.io.File;
+import java.util.Properties;
+
+import org.eclipse.core.runtime.Platform;
+
+public class PropertiesCollectorUtil {
+
+    private static final String ROOT_NODE = "additional_packages_records"; // additional packages
+
+    private static final String PREF_ADDITIONAL_PACKAGES = "AdditionalPackages"; // preference node
+
+    private static final String ADDITONAL_PACKAGE_FILE = "additionalPackages";
+
+
+    public static String getAdditionalPackageRecording() {
+        File file = getRecordFile(ADDITONAL_PACKAGE_FILE);
+
+        Properties props = PropertiesFileUtil.read(file, true);
+        String records = props.getProperty(ROOT_NODE, "");
+
+        return records;
+    }
+
+    public static void storeAdditionalPackageRecording(String records) {
+        File file = getRecordFile(ADDITONAL_PACKAGE_FILE);
+        Properties props = PropertiesFileUtil.read(file, false);
+        props.setProperty(ROOT_NODE, records);
+        PropertiesFileUtil.store(file, props);
+    }
+
+    private static File getRecordFile(String fileName) {
+        String configurationLocation = Platform.getConfigurationLocation().getURL().getPath();
+        File file = new File(configurationLocation + "/data_collector/" + fileName);
+        return file;
+    }
+
+    public static String getAdditionalPackagePreferenceNode() {
+        return PREF_ADDITIONAL_PACKAGES;
+    }
+
+}

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/time/PropertiesFileUtil.java

@@ -0,0 +1,93 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.commons.utils.time;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.talend.commons.exception.CommonExceptionHandler;
+
+/**
+ * DOC sbliu  class global comment. Detailled comment
+ */
+public class PropertiesFileUtil {
+    public static synchronized Properties read(File recordFile, boolean createIfNotExist) {
+        Properties props = new Properties();
+        if (recordFile != null && exist(recordFile, createIfNotExist)) {
+            FileInputStream inStream = null;
+            try {
+                inStream = new FileInputStream(recordFile);
+                props.load(inStream);
+            } catch (Exception e) {
+                CommonExceptionHandler.log(e.getMessage());
+            } finally {
+                if (inStream != null) {
+                    try {
+                        inStream.close();
+                    } catch (IOException e) {//
+                    }
+                }
+            }
+        }
+
+        return props;
+    }
+
+    public static synchronized void store(File recordFile, Properties props) {
+        if (props == null) {
+            return;
+        }
+
+        if (recordFile != null && exist(recordFile, true)) {
+            FileOutputStream outputStream = null;
+            try {
+                outputStream = new FileOutputStream(recordFile);
+                props.store(outputStream, "");
+            } catch (IOException e) {
+                CommonExceptionHandler.log(e.getMessage());
+            } finally {
+                if (outputStream != null) {
+                    try {
+                        outputStream.close();
+                    } catch (IOException e) {
+                        //
+                    }
+                }
+            }
+        }
+    }
+
+    public static boolean exist(File recordFile, boolean createIfNotExist) {
+        boolean exists = recordFile.exists();
+        if (!exists && createIfNotExist) {
+            try {
+                if(!recordFile.getParentFile().exists()) {
+                    recordFile.getParentFile().mkdirs();
+                }
+                exists = recordFile.createNewFile();
+                if (!exists) {
+                    throw new FileNotFoundException(recordFile.getName());
+                }
+            } catch (Exception e) {
+                CommonExceptionHandler.log(e.getMessage());
+                return false;
+            }
+        }
+
+        return exists;
+    }
+}

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/time/TimeMeasurePerformance.java

@@ -0,0 +1,121 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.commons.utils.time;
+
+import java.util.HashMap;
+
+import org.apache.log4j.Logger;
+
+/**
+ * DOC sbliu  class global comment. Detailled comment
+ */
+public class TimeMeasurePerformance extends TimeMeasure{
+    static private Logger logger;
+    
+    private static HashMap<String, TimeStack> timers;
+    
+    private static long startTime = -1L;
+
+    private static int indent = 0;
+    
+    public static void begin(String idTimer, String description) {
+        startTime = System.nanoTime();
+        
+        init();
+        if (timers.containsKey(idTimer)) {
+            log(indent(indent) + "Warning (start): timer " + idTimer + " already exists"); //$NON-NLS-1$  //$NON-NLS-2$
+        } else {
+            indent++;
+            TimeStack times = new TimeStack();
+            timers.put(idTimer, times);
+            
+            String message = "Start '" + idTimer + "' ...";
+            if (description != null) {
+                message = "Start '" + idTimer + "', " + description + " ...";
+            }
+            log(indent(indent) + message); //$NON-NLS-1$  //$NON-NLS-2$
+        }
+    }
+    
+    private static void init() {
+        if (timers == null) {
+            timers = new HashMap<String, TimeStack>();
+        }
+        
+        if(logger == null) {
+            configureLogger();
+        }
+    }
+
+    private static void log (String message) {
+        logger.info(message);
+    }
+    
+    public static long end(String idTimer) {
+        init();
+        if (!timers.containsKey(idTimer)) {
+            log(indent(indent) + "Warning (end): timer " + idTimer + " doesn't exist"); //$NON-NLS-1$  //$NON-NLS-2$
+            return -1;
+        } else {
+            TimeStack timeStack = timers.get(idTimer);
+            timers.remove(idTimer);
+            long elapsedTimeSinceLastRequest = timeStack.getLastStepElapsedTime();
+            log(indent(indent) + "End '" + idTimer + "', elapsed time since last request: " //$NON-NLS-1$  //$NON-NLS-2$
+                    + elapsedTimeSinceLastRequest + " ms "); //$NON-NLS-1$
+            
+            long totalElapsedTime = timeStack.getTotalElapsedTime();
+            
+            log(indent(indent) + "End '" + idTimer + "', total elapsed time: " + totalElapsedTime + " ms "); //$NON-NLS-1$  //$NON-NLS-2$  //$NON-NLS-3$
+            
+            indent--;
+            return totalElapsedTime;
+        }
+    }
+    
+    public static long step(String idTimer, String stepName) {
+        init();
+        if (!timers.containsKey(idTimer)) {
+            log(indent(indent) + "Warning (end): timer " + idTimer + " does'nt exist"); //$NON-NLS-1$  //$NON-NLS-2$
+            return -1;
+        } else {
+            TimeStack timeStack = timers.get(idTimer);
+            timeStack.addStep();
+            /*
+             * trace the timeline of every step,problem is that the code below " Calendar ca = Calendar.getInstance();
+             * Date now = ca.getTime();" will cost almost 13ms~15ms
+             */
+            long time = timeStack.getLastStepElapsedTime();
+            String timerStepName = idTimer + "', step name '" + stepName; //$NON-NLS-1$
+            
+            log(indent(indent)
+                    + "-> '" + timerStepName + "', elapsed time since previous step: " + time + " ms"); //$NON-NLS-1$  //$NON-NLS-2$  //$NON-NLS-3$
+            
+            return time;
+        }
+    }
+    
+    private static void configureLogger() {
+        try {
+            PerformanceLogManager logManager = new PerformanceLogManager();
+            logger = logManager.getLogger(TimeMeasurePerformance.class.getName());
+        } catch (Exception e) {
+            throw new RuntimeException("Error while initializing log properties.", e);
+        }
+    }
+    
+    public static void afterStartup() {
+        double elapsedTimeInSeconds = (double)(System.nanoTime() - startTime)/1000000000;
+        PerformanceStatisticUtil.recordStartupEpapsedTime(elapsedTimeInSeconds);
+        PerformanceStatisticUtil.measureIO();
+    }
+}

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/drawing/background/BackgroundRefresher.java

@@ -240,7 +240,11 @@ public class BackgroundRefresher implements IBackgroundRefresher {
 
             gc.dispose();
 
-            drawableComposite.getBgDrawableComposite().setBackgroundImage(newImage);
+            if (WindowSystem.isBigSurOrLater()) {
+                drawableComposite.refreshBgDrawableCompsite(newImage);
+            } else {
+                drawableComposite.getBgDrawableComposite().setBackgroundImage(newImage);
+            }
 
             clearImage(oldImage);
             oldImage = newImage;

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/drawing/background/IBgDrawableComposite.java

@@ -13,6 +13,7 @@
 package org.talend.commons.ui.swt.drawing.background;
 
 import org.eclipse.swt.graphics.GC;
+import org.eclipse.swt.graphics.Image;
 import org.eclipse.swt.graphics.Point;
 import org.eclipse.swt.widgets.Composite;
 
@@ -28,6 +29,8 @@ public interface IBgDrawableComposite {
 
     public Composite getBgDrawableComposite();
 
+    public void refreshBgDrawableCompsite(Image image);
+
     public void setOffset(Point offsetPoint);
 
     public Point getOffset();

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/formtools/LabelledText.java

@@ -26,6 +26,7 @@ import org.eclipse.swt.widgets.Event;
 import org.eclipse.swt.widgets.Label;
 import org.eclipse.swt.widgets.Listener;
 import org.eclipse.swt.widgets.Text;
+import org.talend.commons.ui.runtime.ws.WindowSystem;
 
 /**
  * Create a Label and a Text.
@@ -153,7 +154,14 @@ public class LabelledText {
         });
         GridDataFactory.swtDefaults().applyTo(label);
 
-        text = new Text(composite, styleField);
+        // For Big Sur, to avoid the password autofill dialog.
+        if (WindowSystem.isBigSurOrLater() && ((styleField & SWT.PASSWORD) != 0)) {
+            styleField ^= SWT.PASSWORD;
+            text = new Text(composite, styleField);
+            text.setEchoChar('*');
+        } else {
+            text = new Text(composite, styleField);
+        }
         text.selectAll(); // enable fast erase use
         int gridDataStyle = SWT.NONE;
         if (isFill) {

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/linking/BgDrawableComposite.java

@@ -12,7 +12,12 @@
 // ============================================================================
 package org.talend.commons.ui.swt.linking;
 
+import org.eclipse.swt.events.DisposeEvent;
+import org.eclipse.swt.events.DisposeListener;
+import org.eclipse.swt.events.PaintEvent;
+import org.eclipse.swt.events.PaintListener;
 import org.eclipse.swt.graphics.GC;
+import org.eclipse.swt.graphics.Image;
 import org.eclipse.swt.graphics.Point;
 import org.eclipse.swt.widgets.Composite;
 import org.talend.commons.ui.swt.drawing.background.IBgDrawableComposite;
@@ -28,6 +33,8 @@ public abstract class BgDrawableComposite implements IBgDrawableComposite {
     private Composite commonParent;
 
     private Point offsetPoint = new Point(0, 0);
+    
+    private Image backgroundImage;
 
     /**
      * DOC amaumont DrawableBackground constructor comment.
@@ -36,9 +43,36 @@ public abstract class BgDrawableComposite implements IBgDrawableComposite {
      */
     public BgDrawableComposite(Composite commonParent) {
         this.commonParent = commonParent;
+        this.commonParent.addPaintListener(new PaintListener() {
+
+            @Override
+            public void paintControl(PaintEvent arg0) {
+                if (backgroundImage != null) {
+                    arg0.gc.drawImage(backgroundImage, 0, 0);
+                }
+            }
+        });
+        this.commonParent.addDisposeListener(new DisposeListener() {
+
+            @Override
+            public void widgetDisposed(DisposeEvent arg0) {
+                if (backgroundImage != null && !backgroundImage.isDisposed()) {
+                    backgroundImage.dispose();
+                }
+            }
+        });
     }
 
-    /*
+    @Override
+    public void refreshBgDrawableCompsite(Image image) {
+        if (backgroundImage != null) {
+            backgroundImage.dispose();
+        }
+        backgroundImage = image;
+        this.commonParent.redraw();
+    }
+
+	/*
      * (non-Javadoc)
      *
      * @see org.talend.commons.ui.swt.drawing.background.IDrawableComposite#drawBackground(org.eclipse.swt.graphics.GC)

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/linking/TreeToTablesLinker.java

@@ -15,6 +15,7 @@ package org.talend.commons.ui.swt.linking;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -77,6 +78,8 @@ public class TreeToTablesLinker<D1, D2> extends BgDrawableComposite implements I
 
     private List<LinkableTable> linkableTableList;
 
+    private HashMap<Table, Point> tableToCommonPointMap = new HashMap<Table, Point>();
+
     /**
      * DOC amaumont TreeToTableLinker constructor comment.
      *
@@ -258,8 +261,34 @@ public class TreeToTablesLinker<D1, D2> extends BgDrawableComposite implements I
 
             Point offset = getOffset();
 
-            gc.drawLine(pointStartStraight.x + offset.x, pointStartStraight.y + offset.y, pointEndStraight.x + offset.x,
-                    pointEndStraight.y + offset.y);
+            if (WindowSystem.isBigSurOrLater()) {
+                if (yStraight < tree.getBounds().y + treeItemHeight) {
+                    yStraight = tree.getBounds().y + treeItemHeight;
+                }
+                if (yStraight > tree.getBounds().height + tree.getBounds().y + treeItemHeight) {
+                    yStraight = tree.getBounds().height + tree.getBounds().y + treeItemHeight;
+                }
+                pointEndStraight = new Point(treeToCommonPoint.x + tree.getClientArea().width, yStraight);
+                if (tableToCommonPointMap.get(table) == null) {
+                    tableToCommonPointMap.put(table, display.map(table, getBgDrawableComposite(), new Point(0, 0)));
+                }
+                // scroll issue for table
+                if (tableToCommonPoint.y != tableToCommonPointMap.get(table).y) {
+                    pointEndCentralCurve.y = tableToCommonPoint.y + tableItemBounds.y 
+                            + table.getItemHeight() / 2;
+                }
+                if (pointEndCentralCurve.y < tableToCommonPointMap.get(table).y) {
+                    pointEndCentralCurve.y = tableToCommonPointMap.get(table).y;
+                }
+                if (pointEndCentralCurve.y > tableToCommonPointMap.get(table).y + tableBounds.height - table.getBorderWidth()
+                        - table.getHeaderHeight()) {
+                    pointEndCentralCurve.y = tableToCommonPointMap.get(table).y + tableBounds.height - table.getBorderWidth()
+                            - table.getHeaderHeight();
+                }
+            } else {
+                gc.drawLine(pointStartStraight.x + offset.x, pointStartStraight.y + offset.y, pointEndStraight.x + offset.x,
+                        pointEndStraight.y + offset.y);
+            }
 
             pointEndStraight.x += offset.x;
             pointEndStraight.y += offset.y;

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/proposal/CommonTextCellEditorWithProposal.java

@@ -212,4 +212,8 @@ public class CommonTextCellEditorWithProposal {
         return this.contentProposalAdapter;
     }
 
+    public int getPreviousActivatedIndex() {
+        return previousActivatedIndex;
+    }
+
 }

main/plugins/org.talend.commons.ui/src/main/java/org/talend/commons/ui/swt/proposal/ExtendedTextCellEditorWithProposal.java

@@ -97,4 +97,7 @@ public class ExtendedTextCellEditorWithProposal extends ExtendedTextCellEditor i
         super.fireCancelEditor();
     }
 
+    public CommonTextCellEditorWithProposal getCommonTextEditor() {
+        return commonTextEditor;
+    }
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/constants/FileConstants.java

@@ -109,4 +109,8 @@ public interface FileConstants {
 
     String SPRING_FOLDER_NAME = "META-INF/spring"; //$NON-NLS-1$
 
+    String TALEND_FOLDER_NAME = "TALEND-INF"; //$NON-NLS-1$
+
+    String MAVEN_FOLDER_NAME = "MAVEN-INF";
+
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/handlers/JobRoutinesItemRelationshipHandler.java

@@ -21,6 +21,7 @@ import org.talend.core.model.properties.Item;
 import org.talend.core.model.relationship.AbstractJobItemRelationshipHandler;
 import org.talend.core.model.relationship.Relation;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
+import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
 import org.talend.designer.core.model.utils.emf.talendfile.RoutinesParameterType;
 
@@ -47,14 +48,24 @@ public class JobRoutinesItemRelationshipHandler extends AbstractJobItemRelations
             Map<String, String> currentSystemRoutinesMap = RelationshipItemBuilder.getInstance().getCurrentSystemRoutinesMap();
             for (Object o : processType.getParameters().getRoutinesParameter()) {
                 RoutinesParameterType itemInfor = (RoutinesParameterType) o;
-                if (currentSystemRoutinesMap.containsValue(itemInfor.getName())) {
+                if (itemInfor.getName() != null && currentSystemRoutinesMap.containsValue(itemInfor.getName())) {
                     // exclude system routines relation
                     continue;
                 }
 
                 Relation addedRelation = new Relation();
-                addedRelation.setId(itemInfor.getName());
-                addedRelation.setType(RelationshipItemBuilder.ROUTINE_RELATION);
+                if (ERepositoryObjectType.ROUTINESJAR != null
+                        && ERepositoryObjectType.ROUTINESJAR.getType().equals(itemInfor.getType())) {
+                    addedRelation.setId(itemInfor.getId());
+                    addedRelation.setType(RelationshipItemBuilder.ROUTINES_JAR_RELATION);
+                } else if (ERepositoryObjectType.BEANSJAR != null
+                        && ERepositoryObjectType.BEANSJAR.getType().equals(itemInfor.getType())) {
+                    addedRelation.setId(itemInfor.getId());
+                    addedRelation.setType(RelationshipItemBuilder.BEANS_JAR_RELATION);
+                } else {
+                    addedRelation.setId(itemInfor.getName());
+                    addedRelation.setType(RelationshipItemBuilder.ROUTINE_RELATION);
+                }
                 addedRelation.setVersion(RelationshipItemBuilder.LATEST_VERSION);
                 relationSet.add(addedRelation);
 

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/logintask/SyncLibrariesLoginTask.java

@@ -67,4 +67,9 @@ public class SyncLibrariesLoginTask extends AbstractLoginTask implements IRunnab
                 });
     }
 
+    @Override
+    public boolean isRequiredAlways() {
+        return true;
+    }
+
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/AbstractEMFRepositoryFactory.java

@@ -837,8 +837,8 @@ public abstract class AbstractEMFRepositoryFactory extends AbstractRepositoryFac
         Object fullFolder = getFullFolder(project, type, relativeFolder);
         serializableAllVersion = getSerializableFromFolder(project, fullFolder, id, type, false, false, true, true);
         if (serializableAllVersion.isEmpty()) {
-            // look in all folders
-            serializableAllVersion = getSerializable(project, id, false, false);
+            // look in all folders for this item type
+            serializableAllVersion = getSerializableFromFolder(project, fullFolder, id, type, false, true, true, true, true);
         }
         int size = serializableAllVersion.size();
 

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/IRepositoryFactory.java

@@ -15,6 +15,7 @@ package org.talend.core.repository.model;
 import java.util.List;
 
 import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
@@ -186,7 +187,7 @@ public interface IRepositoryFactory {
      * @throws PersistenceException
      */
     public List<IRepositoryViewObject> getAll(Project project, ERepositoryObjectType type, boolean withDeleted,
-            boolean allVersions) throws PersistenceException;
+            boolean allVersions, IFolder... folders) throws PersistenceException;
 
     /**
      * Deletes logically the given object. <code>isDeleted</code> on this object will now returned <code>true</code>.
@@ -270,6 +271,8 @@ public interface IRepositoryFactory {
 
     public void create(Project project, Item item, IPath path, boolean... isImportItem) throws PersistenceException;
 
+    public void save(Project project, Item item, boolean isMigrationTask) throws PersistenceException;
+    
     public void save(Project project, Item item) throws PersistenceException;
 
     public void save(Project project, Property property) throws PersistenceException;

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProjectRepositoryNode.java

@@ -20,6 +20,7 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import org.apache.commons.lang.ArrayUtils;
@@ -675,6 +676,9 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
             }
         }
 
+        List<IRepositoryViewObject> codeObjectList = new ArrayList<IRepositoryViewObject>();
+        Map<String, List<RepositoryNode>> labelCodeJarMap = new HashMap<String, List<RepositoryNode>>();
+
         for (IRepositoryViewObject currentObject : elements) {
             RepositoryNode parent = getFolder(currentObject.getRepositoryObjectType(), currentObject.getPath(), rootNodes);
             RepositoryNode parentNode = parent;
@@ -682,20 +686,69 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
                 parentNode = rootNode;
             }
             if (currentObject.isDeleted()) {
+                boolean isCodeJarType = false;
+                ERepositoryObjectType repositoryObjectType = currentObject.getRepositoryObjectType();
+                if (ERepositoryObjectType.getAllTypesOfCodes().contains(repositoryObjectType)) {
+                    codeObjectList.add(currentObject);
+                    continue;
+                } else if (ERepositoryObjectType.getAllTypesOfCodesJar().contains(repositoryObjectType)) {
+                    isCodeJarType = true;
+                }
+
                 RepositoryNode repNode = new RepositoryNode(new RepositoryViewObject(currentObject.getProperty()), parentNode,
                         ENodeType.REPOSITORY_ELEMENT);
                 repNode.setProperties(EProperties.CONTENT_TYPE, currentObject.getRepositoryObjectType());
                 repNode.setProperties(EProperties.LABEL, currentObject.getLabel());
                 parentNode.getChildren().add(repNode);
                 repNode.setParent(parentNode);
+
+                if (isCodeJarType) {
+                    String label = currentObject.getProperty().getLabel();
+                    if (labelCodeJarMap.get(label) == null) {
+                        labelCodeJarMap.put(label, new ArrayList<RepositoryNode>());
+                    }
+                    labelCodeJarMap.get(label).add(repNode);
+                }
             } else {
                 addDeletedSubItems(currentObject.getProperty().getItem(), parentNode);
             }
         }
+        addDeletedCode(rootNode, codeObjectList, labelCodeJarMap);
         objects.removeAll(elements);
         addDeletedTestCases(rootNode, objects);
     }
 
+    private void addDeletedCode(RepositoryNode rootNode, List<IRepositoryViewObject> codeObjectList, Map<String, List<RepositoryNode>> labelCodeJarMap) {
+        List<IRepositoryNode> rootNodes = rootNode.getChildren();
+        for (IRepositoryViewObject codeObject : codeObjectList) {
+            ERepositoryObjectType objectType = codeObject.getRepositoryObjectType();
+            RepositoryNode parent = getFolder(codeObject.getRepositoryObjectType(), codeObject.getPath(), rootNodes);
+            RepositoryNode parentNode = parent;
+            if (parentNode == null) {
+                parentNode = rootNode;
+            }
+
+            Item parentItem = (Item) codeObject.getProperty().getItem().getParent();
+            String folderItemLabel = parentItem.getProperty().getLabel();
+            if (labelCodeJarMap.get(folderItemLabel) != null) {
+                Optional node = labelCodeJarMap.get(folderItemLabel).stream()
+                        .filter(n -> ERepositoryObjectType.CodeTypeEnum.isCodeRepositoryObjectTypeMatch(n.getObjectType(),
+                                objectType))
+                        .findFirst();
+                if (node.isPresent()) {
+                    parentNode = (RepositoryNode) node.get();
+                }
+            }
+
+            RepositoryNode repNode = new RepositoryNode(new RepositoryViewObject(codeObject.getProperty()), parentNode,
+                    ENodeType.REPOSITORY_ELEMENT);
+            repNode.setProperties(EProperties.CONTENT_TYPE, objectType);
+            repNode.setProperties(EProperties.LABEL, codeObject.getLabel());
+            parentNode.getChildren().add(repNode);
+            repNode.setParent(parentNode);
+        }
+    }
+
     /**
      * DOC nrousseau Comment method "addDeletedElements".
      *
@@ -1064,6 +1117,9 @@ public class ProjectRepositoryNode extends RepositoryNode implements IProjectRep
         }
         if (newProject != null && newProject.getEmfProject() != null) {
             List<FolderItem> folderItems = ProjectManager.getInstance().getFolders(newProject.getEmfProject());
+            if (folderItems != null) {
+                folderItems = new ArrayList<>(folderItems);
+            }
             for (FolderItem folder : folderItems) {
                 String folderName = folder.getProperty().getLabel();
                 if (("process".equals(folderName) || "joblets".equals(folderName)) && folder.getChildren() != null

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java

@@ -15,6 +15,7 @@ package org.talend.core.repository.model;
 import java.beans.PropertyChangeListener;
 import java.beans.PropertyChangeSupport;
 import java.io.File;
+import java.lang.reflect.InvocationTargetException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Date;
@@ -24,6 +25,7 @@ import java.util.Hashtable;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
@@ -71,7 +73,7 @@ import org.talend.commons.ui.gmf.util.DisplayUtils;
 import org.talend.commons.ui.runtime.exception.MessageBoxExceptionHandler;
 import org.talend.commons.utils.data.container.RootContainer;
 import org.talend.commons.utils.network.TalendProxySelector;
-import org.talend.commons.utils.time.TimeMeasure;
+import org.talend.commons.utils.time.TimeMeasurePerformance;
 import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.AbstractDQModelService;
 import org.talend.core.GlobalServiceRegister;
@@ -84,6 +86,7 @@ import org.talend.core.context.Context;
 import org.talend.core.context.RepositoryContext;
 import org.talend.core.exception.TalendInternalPersistenceException;
 import org.talend.core.hadoop.BigDataBasicUtil;
+import org.talend.core.model.general.ILibrariesService;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.Project;
 import org.talend.core.model.metadata.MetadataTalendType;
@@ -104,6 +107,7 @@ import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.ProjectReference;
 import org.talend.core.model.properties.PropertiesPackage;
 import org.talend.core.model.properties.Property;
+import org.talend.core.model.properties.RoutineItem;
 import org.talend.core.model.properties.SpagoBiServer;
 import org.talend.core.model.properties.Status;
 import org.talend.core.model.properties.User;
@@ -119,23 +123,29 @@ import org.talend.core.model.repository.LockInfo;
 import org.talend.core.model.repository.RepositoryContentManager;
 import org.talend.core.model.repository.RepositoryObject;
 import org.talend.core.model.repository.RepositoryViewObject;
+import org.talend.core.model.routines.CodesJarInfo;
 import org.talend.core.repository.CoreRepositoryPlugin;
 import org.talend.core.repository.constants.Constant;
 import org.talend.core.repository.constants.FileConstants;
 import org.talend.core.repository.i18n.Messages;
 import org.talend.core.repository.recyclebin.RecycleBinManager;
+import org.talend.core.repository.utils.LoginTaskRegistryReader;
 import org.talend.core.repository.utils.ProjectDataJsonProvider;
 import org.talend.core.repository.utils.RepositoryPathProvider;
 import org.talend.core.repository.utils.XmiResourceManager;
 import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.runtime.repository.item.ItemProductKeys;
+import org.talend.core.runtime.services.IGenericWizardService;
 import org.talend.core.runtime.services.IMavenUIService;
 import org.talend.core.runtime.util.ItemDateParser;
+import org.talend.core.runtime.util.SharedStudioUtils;
 import org.talend.core.service.ICoreUIService;
 import org.talend.core.service.IUpdateService;
+import org.talend.core.utils.CodesJarResourceCache;
 import org.talend.cwm.helper.SubItemHelper;
 import org.talend.cwm.helper.TableHelper;
 import org.talend.designer.runprocess.IRunProcessService;
+import org.talend.login.ILoginTask;
 import org.talend.repository.ProjectManager;
 import org.talend.repository.ReferenceProjectProblemManager;
 import org.talend.repository.ReferenceProjectProvider;
@@ -177,6 +187,8 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
 
     private boolean isCancelled;
 
+    private static final LoginTaskRegistryReader LOGIN_TASK_REGISTRY_READER = new LoginTaskRegistryReader();
+
     @Override
     public synchronized void addPropertyChangeListener(PropertyChangeListener l) {
         if (l == null) {
@@ -226,6 +238,13 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
         return null;
     }
 
+    private ILibrariesService getLibrariesService() {
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibrariesService.class)) {
+            return GlobalServiceRegister.getDefault().getService(ILibrariesService.class);
+        }
+        return null;
+    }
+    
     /*
      * (non-Javadoc)
      *
@@ -1213,6 +1232,28 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
         return this.repositoryFactoryFromProvider.getLastVersion(project, ProcessUtils.getPureItemId(id), folderPath, type);
     }
 
+
+    @Override
+    public IRepositoryViewObject getLastVersion(String id, ERepositoryObjectType type)
+            throws PersistenceException {
+        return getLastVersion(id , "", type);
+    }
+    
+    @Override
+    public IRepositoryViewObject getLastVersion(String id, List<ERepositoryObjectType> types) throws PersistenceException {
+        if (types != null) {
+            IRepositoryViewObject object = null;
+            for (ERepositoryObjectType type : types) {
+                object = getLastVersion(id, type);
+                if (object != null) {
+                    return object;
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
     public IRepositoryViewObject getLastVersion(String id, String folderPath, ERepositoryObjectType type)
             throws PersistenceException {
         String objId = id;
@@ -1224,7 +1265,25 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 return this.repositoryFactoryFromProvider.getLastVersion(project, objId, folderPath, type);
             }
         }
-        return this.repositoryFactoryFromProvider.getLastVersion(projectManager.getCurrentProject(), objId , folderPath, type);
+        return getLastRefVersion(projectManager.getCurrentProject(), objId , folderPath, type);
+    }
+    
+    @Override
+    public IRepositoryViewObject getLastRefVersion(Project project, String id, String folderPath, ERepositoryObjectType type) throws PersistenceException {
+        String projectLabel = ProcessUtils.getProjectLabelFromItemId(id);
+        IRepositoryViewObject lastVersion = getLastVersion(project, ProcessUtils.getPureItemId(id), folderPath, type);
+        if (lastVersion == null) {
+            for (Project p : projectManager.getReferencedProjects(project)) {
+                if (projectLabel != null && !projectLabel.equals(p.getTechnicalLabel())) {
+                    continue;
+                }
+                lastVersion = getLastRefVersion(p, id);
+                if (lastVersion != null) {
+                    break;
+                }
+            }
+        }
+        return lastVersion;
     }
 
     @Override
@@ -1293,6 +1352,35 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
         return this.repositoryFactoryFromProvider.getAll(projectManager.getCurrentProject(), type, withDeleted, allVersions);
     }
 
+    @Override
+    public List<IRepositoryViewObject> getAll(Project project, ERepositoryObjectType type, boolean withDeleted,
+            boolean allVersions, IFolder... folders) throws PersistenceException {
+        return this.repositoryFactoryFromProvider.getAll(project, type, withDeleted, allVersions, folders);
+    }
+
+    @Override
+    public List<IRepositoryViewObject> getAllCodesJars(ERepositoryObjectType type) throws PersistenceException {
+        return getAllCodesJars(projectManager.getCurrentProject(), type);
+    }
+
+    @Override
+    public List<IRepositoryViewObject> getAllCodesJars(Project project, ERepositoryObjectType type) throws PersistenceException {
+        return getAll(project, type).stream().filter(obj -> !(obj.getProperty().getItem() instanceof RoutineItem))
+                .collect(Collectors.toList());
+    }
+
+    @Override
+    public List<IRepositoryViewObject> getAllInnerCodes(CodesJarInfo info) throws PersistenceException {
+        Project project = ProjectManager.getInstance().getProjectFromProjectTechLabel(info.getProjectTechName());
+        // empty folder won't be commit in git, create if not exist
+        IFolder folder = ResourceUtils.getProject(project).getFolder(ERepositoryObjectType.getFolderName(info.getType()))
+                .getFolder(info.getLabel());
+        if (!folder.exists()) {
+            ResourceUtils.createFolder(folder);
+        }
+        return repositoryFactoryFromProvider.getAll(project, info.getType(), false, false, folder);
+    }
+
     @Override
     public List<String> getFolders(ERepositoryObjectType type) throws PersistenceException {
         return getFolders(projectManager.getCurrentProject(), type);
@@ -1546,14 +1634,15 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
 
     @Override
     public void save(Project project, Item item, boolean... isMigrationTask) throws PersistenceException {
-        this.repositoryFactoryFromProvider.save(project, item);
         if (isMigrationTask == null || isMigrationTask.length == 0 || !isMigrationTask[0]) {
+            this.repositoryFactoryFromProvider.save(project, item);
             boolean avoidGenerateProm = false;
             if (isMigrationTask != null && isMigrationTask.length == 2) {
                 avoidGenerateProm = isMigrationTask[1];
             }
             fireRepositoryPropertyChange(ERepositoryActionName.SAVE.getName(), avoidGenerateProm, item);
-
+        } else {
+            this.repositoryFactoryFromProvider.save(project, item, true);
         }
     }
 
@@ -1744,9 +1833,12 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
      * @throws PersistenceException
      */
     private void emptyTempFolder(Project project) throws PersistenceException {
-        String str = (System.getProperty("eclipse.home.location") + "temp").substring(5);
-        FilesUtils.deleteFolder(new File(str), false);
-
+    	try {
+            String str = SharedStudioUtils.getTempFolderPath().toPortableString();
+            FilesUtils.deleteFolder(new File(str), false);
+    	}catch (Exception ex) {
+    		ExceptionHandler.process(ex);
+    	}
         long start = System.currentTimeMillis();
         IProject fsProject = ResourceUtils.getProject(project);
         IFolder folder = ResourceUtils.getFolder(fsProject, RepositoryConstants.TEMP_DIRECTORY, false);
@@ -2035,11 +2127,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
      */
     public void logOnProject(Project project, IProgressMonitor monitor) throws LoginException, PersistenceException {
         try {
-            TimeMeasure.display = CommonsPlugin.isDebugMode();
-            TimeMeasure.displaySteps = CommonsPlugin.isDebugMode();
-            TimeMeasure.measureActive = CommonsPlugin.isDebugMode();
-
-            TimeMeasure.begin("logOnProject"); //$NON-NLS-1$
+            TimeMeasurePerformance.begin("logOnProject", "logon project name '" + project.getLabel()+"'"); //$NON-NLS-1$ //$NON-NLS-2$
             try {
                 /**
                  * init/check proxy selector, in case default proxy selector is not registed yet
@@ -2083,12 +2171,35 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 this.repositoryFactoryFromProvider.beforeLogon(project);
                 ProjectManager.getInstance().getBeforeLogonRecords().clear();
                 ProjectManager.getInstance().getUpdatedRemoteHandlerRecords().clear();
+                ILibrariesService librariesService = getLibrariesService();
+                if (librariesService != null) {
+                	librariesService.setForceReloadCustomUri();
+                }
 
                 ProjectDataJsonProvider.checkAndRectifyRelationShipSetting(project.getEmfProject());
 
+                try {
+                    // load additional jdbc
+                    if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericWizardService.class)) {
+                        IGenericWizardService service = GlobalServiceRegister.getDefault()
+                                .getService(IGenericWizardService.class);
+                        if (service != null) {
+                            service.loadAdditionalJDBC();
+                        }
+                    }
+                } catch (Exception e) {
+                    // in case, to avoid block logon
+                    ExceptionHandler.process(e);
+                }
+
                 // init dynamic distirbution after `beforeLogon`, before loading libraries.
                 initDynamicDistribution(monitor);
 
+                if (GlobalServiceRegister.getDefault().isServiceRegistered(IUpdateService.class)) {
+                    IUpdateService updateService = GlobalServiceRegister.getDefault().getService(IUpdateService.class);
+                    updateService.syncComponentM2Jars(currentMonitor);
+                }
+
                 // init sdk component
                 try {
                     currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE);
@@ -2106,7 +2217,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                     throw new OperationCanceledException(""); //$NON-NLS-1$
                 }
                 // monitorWrap.worked(1);
-                TimeMeasure.step("logOnProject", "beforeLogon"); //$NON-NLS-1$ //$NON-NLS-2$
+                TimeMeasurePerformance.step("logOnProject", "beforeLogon"); //$NON-NLS-1$ //$NON-NLS-2$
 
                 // Check project compatibility
                 checkProjectCompatibility(project);
@@ -2124,16 +2235,26 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                     currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE);
                     currentMonitor.beginTask(Messages.getString("ProxyRepositoryFactory.synchronizeLibraries"), 1); //$NON-NLS-1$
                     coreService.syncLibraries(currentMonitor);
-                    TimeMeasure.step("logOnProject", "Sync components libraries"); //$NON-NLS-1$
+                    TimeMeasurePerformance.step("logOnProject", "Sync components libraries"); //$NON-NLS-1$
                 }
 
+                try {
+                    // for new added mapping file, sync to project mapping folder
+                    MetadataTalendType.syncNewMappingFileToProject();
+                } catch (SystemException e) {
+                    // ignore
+                    ExceptionHandler.process(e);
+                }
+
+                CodesJarResourceCache.initCodesJarCache();
+
                 currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE);
                 currentMonitor.beginTask("Execute before logon migrations tasks", 1); //$NON-NLS-1$
                 ProjectManager.getInstance().getMigrationRecords().clear();
                 executeMigrations(project, true, currentMonitor);
                 ProjectManager.getInstance().getMigrationRecords().clear();
                 // monitorWrap.worked(1);
-                TimeMeasure.step("logOnProject", "executeMigrations(beforeLogonTasks)"); //$NON-NLS-1$ //$NON-NLS-2$
+                TimeMeasurePerformance.step("logOnProject", "executeMigrations(beforeLogonTasks)"); //$NON-NLS-1$ //$NON-NLS-2$
 
                 currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE);
                 currentMonitor.beginTask(Messages.getString("ProxyRepositoryFactory.logonInProgress"), 1); //$NON-NLS-1$
@@ -2141,7 +2262,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 this.repositoryFactoryFromProvider.logOnProject(project);
                 ProjectManager.getInstance().getLogonRecords().clear();
                 // monitorWrap.worked(1);
-                TimeMeasure.step("logOnProject", "logOnProject"); //$NON-NLS-1$ //$NON-NLS-2$
+                TimeMeasurePerformance.step("logOnProject", "logOnProject"); //$NON-NLS-1$ //$NON-NLS-2$
 
                 emptyTempFolder(project);
 
@@ -2157,11 +2278,6 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
 
                 fireRepositoryPropertyChange(ERepositoryActionName.PROJECT_PREFERENCES_RELOAD.getName(), null, null);
 
-                if (GlobalServiceRegister.getDefault().isServiceRegistered(IUpdateService.class)) {
-                    IUpdateService updateService = GlobalServiceRegister.getDefault().getService(IUpdateService.class);
-                    updateService.syncComponentM2Jars(currentMonitor);
-                }
-
                 IRunProcessService runProcessService = getRunProcessService();
                 if (runProcessService != null) {
                     runProcessService.initMavenJavaProject(monitor, project);
@@ -2172,7 +2288,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 ProjectManager.getInstance().getMigrationRecords().clear();
                 executeMigrations(project, false, currentMonitor);
                 ProjectManager.getInstance().getMigrationRecords().clear();
-                TimeMeasure.step("logOnProject", "executeMigrations(afterLogonTasks)"); //$NON-NLS-1$ //$NON-NLS-2$
+                TimeMeasurePerformance.step("logOnProject", "executeMigrations(afterLogonTasks)"); //$NON-NLS-1$ //$NON-NLS-2$
                 if (monitor != null && monitor.isCanceled()) {
                     throw new OperationCanceledException(""); //$NON-NLS-1$
                 }
@@ -2188,12 +2304,12 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                     // clean workspace
                     currentMonitor.beginTask(Messages.getString("ProxyRepositoryFactory.cleanWorkspace"), 1); //$NON-NLS-1$
 
-                    TimeMeasure.step("logOnProject", "clean Java project"); //$NON-NLS-1$ //$NON-NLS-2$
+                    TimeMeasurePerformance.step("logOnProject", "clean Java project"); //$NON-NLS-1$ //$NON-NLS-2$
 
                     if (workspace instanceof Workspace) {
                         ((Workspace) workspace).getFileSystemManager().getHistoryStore().clean(currentMonitor);
                     }
-                    TimeMeasure.step("logOnProject", "clean workspace history"); //$NON-NLS-1$ //$NON-NLS-2$
+                    TimeMeasurePerformance.step("logOnProject", "clean workspace history"); //$NON-NLS-1$ //$NON-NLS-2$
 
                     currentMonitor = subMonitor.newChild(1, SubMonitor.SUPPRESS_NONE);
                     currentMonitor.beginTask(Messages.getString("ProxyRepositoryFactory.synch.repo.items"), 1); //$NON-NLS-1$
@@ -2219,12 +2335,12 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 if (monitor != null && monitor.isCanceled()) {
                     throw new OperationCanceledException(""); //$NON-NLS-1$
                 }
-                TimeMeasure.step("logOnProject", "sync repository (routines/rules/beans)"); //$NON-NLS-1$ //$NON-NLS-2$
+                TimeMeasurePerformance.step("logOnProject", "sync repository (routines/rules/beans)"); //$NON-NLS-1$ //$NON-NLS-2$
 
                 // log4j prefs
                 if (coreUiService != null && coreService != null) {
                     coreService.syncLog4jSettings(null);
-                    TimeMeasure.step("logOnProject", "sync log4j"); //$NON-NLS-1$ //$NON-NLS-2$
+                    TimeMeasurePerformance.step("logOnProject", "sync log4j"); //$NON-NLS-1$ //$NON-NLS-2$
                 }
 
                 try {
@@ -2241,7 +2357,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 if (runProcessService != null && !isCommandLineLocalRefProject) {
                     runProcessService.initializeRootPoms(monitor);
 
-                    TimeMeasure.step("logOnProject", "install / setup root poms"); //$NON-NLS-1$ //$NON-NLS-2$
+                    TimeMeasurePerformance.step("logOnProject", "install / setup root poms"); //$NON-NLS-1$ //$NON-NLS-2$
                 }
                 if (GlobalServiceRegister.getDefault().isServiceRegistered(ITDQRepositoryService.class)) {
                     ITDQRepositoryService tdqRepositoryService = GlobalServiceRegister.getDefault()
@@ -2258,10 +2374,7 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
                 fullLogonFinished = true;
                 this.repositoryFactoryFromProvider.afterLogon(monitor);
             } finally {
-                TimeMeasure.end("logOnProject"); //$NON-NLS-1$
-                TimeMeasure.display = false;
-                TimeMeasure.displaySteps = false;
-                TimeMeasure.measureActive = false;
+                TimeMeasurePerformance.end("logOnProject"); //$NON-NLS-1$
             }
             String str[] = new String[] { getRepositoryContext().getUser() + "", projectManager.getCurrentProject() + "" }; //$NON-NLS-1$ //$NON-NLS-2$
             log.info(Messages.getString("ProxyRepositoryFactory.log.loggedOn", str)); //$NON-NLS-1$
@@ -2662,4 +2775,13 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
     public RepositoryWorkUnit getWorkUnitInProgress() {
         return repositoryFactoryFromProvider.getWorkUnitInProgress();
     }
+
+    public void executeRequiredLoginTasks(IProgressMonitor monitor) throws InvocationTargetException, InterruptedException {
+        ILoginTask[] allLoginTasks = LOGIN_TASK_REGISTRY_READER.getAllTaskListInstance();
+        for (ILoginTask task : allLoginTasks) {
+            if (task.isRequiredAlways()) {
+                task.run(monitor);
+            }
+        }
+    }
 }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/recyclebin/RecycleBinManager.java

@@ -14,6 +14,7 @@ package org.talend.core.repository.recyclebin;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
@@ -24,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.StringUtils;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.emf.common.util.EList;
@@ -275,6 +277,38 @@ public class RecycleBinManager {
                 resource = createRecycleBinResource(project);
             }
             resource.getContents().clear();
+            EList<String> deletedFolders = recycleBin.getDeletedFolders();
+            if (deletedFolders != null) {
+                List<String> folders = new LinkedList<>(deletedFolders);
+                Collections.sort(folders);
+                deletedFolders.clear();
+                deletedFolders.addAll(folders);
+            }
+            EList<TalendItem> deletedItems = recycleBin.getDeletedItems();
+            if (deletedItems != null) {
+                List<TalendItem> items = new LinkedList<>(deletedItems);
+                items.sort((l, r) -> {
+                    if (l == null && r == null) {
+                        return 0;
+                    } else if (l == null) {
+                        return -1;
+                    } else if (r == null) {
+                        return 1;
+                    }
+                    int result = StringUtils.compare(l.getType(), r.getType());
+                    if (result != 0) {
+                        return result;
+                    }
+                    result = StringUtils.compare(l.getPath(), r.getPath());
+                    if (result != 0) {
+                        return result;
+                    }
+                    return StringUtils.compare(l.getId(), r.getId());
+                });
+                deletedItems.clear();
+                deletedItems.addAll(items);
+            }
+
             // set date to null to avoid timezone conflict
             recycleBin.setLastUpdate(null);
             resource.getContents().add(recycleBin);

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/CopyObjectAction.java

@@ -36,7 +36,6 @@ import org.eclipse.swt.widgets.Display;
 import org.talend.commons.exception.BusinessException;
 import org.talend.commons.exception.LoginException;
 import org.talend.commons.exception.PersistenceException;
-import org.talend.commons.exception.SystemException;
 import org.talend.commons.runtime.model.repository.ERepositoryStatus;
 import org.talend.commons.ui.runtime.exception.ExceptionHandler;
 import org.talend.core.GlobalServiceRegister;
@@ -54,11 +53,16 @@ import org.talend.core.model.properties.SQLPatternItem;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
+import org.talend.core.model.routines.CodesJarInfo;
+import org.talend.core.model.routines.RoutinesUtil;
 import org.talend.core.repository.model.ProxyRepositoryFactory;
 import org.talend.core.repository.ui.dialog.PastSelectorDialog;
+import org.talend.core.repository.utils.RoutineUtils;
+import org.talend.core.runtime.services.IDesignerMavenService;
 import org.talend.core.runtime.services.IGenericDBService;
 import org.talend.core.ui.ICDCProviderService;
 import org.talend.core.ui.ITestContainerProviderService;
+import org.talend.core.utils.CodesJarResourceCache;
 import org.talend.designer.codegen.ICodeGeneratorService;
 import org.talend.repository.RepositoryWorkUnit;
 import org.talend.repository.model.IProxyRepositoryFactory;
@@ -125,6 +129,44 @@ public class CopyObjectAction {
         // return false;
         // }
 
+        // disable copy for codejar
+        ERepositoryObjectType sourceObjectType = objectToCopy.getRepositoryObjectType();
+        if (ERepositoryObjectType.getAllTypesOfCodesJar().contains(sourceObjectType)) {
+            return false;
+        }
+
+        // disable paste between global code and custom code
+        if (ERepositoryObjectType.getAllTypesOfCodes().contains(sourceObjectType) && targetNode != null) {
+            IRepositoryViewObject targetObject = targetNode.getObject();
+            boolean isInnerCodeSourceNode = RoutinesUtil.isInnerCodes(objectToCopy.getProperty());
+            if (targetObject != null) {
+                if (sourceObjectType.equals(targetObject.getRepositoryObjectType())) {
+                    return isInnerCodeSourceNode && RoutinesUtil.isInnerCodes(targetObject.getProperty());
+                }
+
+                if (isInnerCodeSourceNode
+                        && !ERepositoryObjectType.getAllTypesOfCodesJar().contains(targetObject.getRepositoryObjectType())) {
+                    return false;
+                }
+                if (!isInnerCodeSourceNode
+                        && ERepositoryObjectType.getAllTypesOfCodesJar().contains(targetObject.getRepositoryObjectType())) {
+                    return false;
+                }
+            } else {
+                // system folder node
+                if (isInnerCodeSourceNode) {
+                    return false;
+                } else {
+                    ERepositoryObjectType targetType = ((ERepositoryObjectType) targetNode
+                            .getProperties(EProperties.CONTENT_TYPE));
+                    if (ERepositoryObjectType.getAllTypesOfCodesJar().contains(targetType)) {
+                        return false;
+                    }
+                }
+
+            }
+        }
+
         // Cannot copy system routines:
         if (objectToCopy.getRepositoryObjectType() == ERepositoryObjectType.ROUTINES) {
             Property property = objectToCopy.getProperty();
@@ -193,7 +235,8 @@ public class CopyObjectAction {
         }
 
         // for bug 0005454: Copy paste with keyboard in the repository view doesn't work.
-        if (targetNode.getType() == ENodeType.REPOSITORY_ELEMENT) {
+        if (targetNode.getType() == ENodeType.REPOSITORY_ELEMENT
+                && !ERepositoryObjectType.getAllTypesOfCodesJar().contains(targetNode.getObjectType())) {
             targetNode = targetNode.getParent();
         }
 
@@ -471,9 +514,23 @@ public class CopyObjectAction {
                 ICodeGeneratorService.class);
         if (codeGenService != null) {
             codeGenService.createRoutineSynchronizer().renameRoutineClass(item);
+            boolean isInnerCode = RoutinesUtil.isInnerCodes(item.getProperty());
+            if (isInnerCode) {
+                RoutineUtils.changeInnerCodePackage(item, true);
+            }
             try {
                 codeGenService.createRoutineSynchronizer().syncRoutine(item, true);
-            } catch (SystemException e) {
+
+                if (isInnerCode) {
+                    CodesJarInfo info = CodesJarResourceCache.getCodesJarByInnerCode(item);
+                    if (info != null) {
+                        IDesignerMavenService designerMavenService = IDesignerMavenService.get();
+                        if (designerMavenService != null) {
+                            designerMavenService.updateCodeJarMavenProject(info, false);
+                        }
+                    }
+                }
+            } catch (Exception e) {
                 ExceptionHandler.process(e);
             }
         }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/CreateFolderAction.java

@@ -111,7 +111,8 @@ public class CreateFolderAction extends AContextualAction {
                         || property.equals(ERepositoryObjectType.JOB_DOC) || property.equals(ERepositoryObjectType.JOBLET_DOC)
                         || ERepositoryObjectType.SQLPATTERNS.equals(property)
                         || ERepositoryObjectType.REFERENCED_PROJECTS.equals(property)
-                        || ERepositoryObjectType.SVN_ROOT.equals(property)) {
+                        || ERepositoryObjectType.SVN_ROOT.equals(property) || ERepositoryObjectType.ROUTINESJAR.equals(property)
+                        || property.equals(ERepositoryObjectType.BEANSJAR)) {
                     canWork = false;
                 } else if (property != null
                         && GlobalServiceRegister.getDefault().isServiceRegistered(ICamelDesignerCoreService.class)) {

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/DeleteAction.java

@@ -21,6 +21,7 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.collections.map.MultiKeyMap;
+import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.resources.IWorkspace;
 import org.eclipse.core.resources.IWorkspaceRunnable;
 import org.eclipse.core.resources.ResourcesPlugin;
@@ -54,6 +55,7 @@ import org.talend.commons.ui.runtime.exception.MessageBoxExceptionHandler;
 import org.talend.commons.ui.runtime.image.EImage;
 import org.talend.commons.ui.runtime.image.ImageProvider;
 import org.talend.commons.utils.data.container.RootContainer;
+import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.IESBService;
 import org.talend.core.ITDQRepositoryService;
@@ -714,6 +716,7 @@ public class DeleteAction extends AContextualAction {
             removeConnFromSQLExplorer(repositoryNode);
             List<IRepositoryViewObject> batchDeleteObjectList = new ArrayList<IRepositoryViewObject>();
             deleteTestCases(factory, deleteActionCache, repositoryNode, null, batchDeleteObjectList);
+            deleteCodeSubItem(factory, deleteActionCache, repositoryNode, null, true);
 
             return true;
         }
@@ -1456,10 +1459,11 @@ public class DeleteAction extends AContextualAction {
                             ITestContainerProviderService testService = (ITestContainerProviderService) GlobalServiceRegister.getDefault()
                                     .getService(ITestContainerProviderService.class);
                             if(testService != null){
-
+                                testService.deleteDataFiles(objToDelete);
                             }
-                            testService.deleteDataFiles(objToDelete);
                         }
+                        // delete codejar forever need to delete subitem also
+                        deleteCodeSubItem(factory, deleteActionCache, currentJobNode, confirm, false);
 
                         if (!ProjectManager.getInstance().getCurrentProject().isLocal()) {
                             // if remote,batch delete later
@@ -1482,14 +1486,47 @@ public class DeleteAction extends AContextualAction {
                 factory.deleteObjectLogical(objToDelete);
                 updateRelatedViews();
                 removeConnFromSQLExplorer(currentJobNode);
-                List<IRepositoryViewObject> deleteObjectList = new ArrayList<IRepositoryViewObject>();
                 deleteTestCases(factory, deleteActionCache, currentJobNode, confirm, null);
+                deleteCodeSubItem(factory, deleteActionCache, currentJobNode, confirm, true);
             }
         }
 
         return needReturn;
     }
 
+    private void deleteCodeSubItem(IProxyRepositoryFactory factory, DeleteActionCache deleteActionCache,
+            final IRepositoryNode currentJobNode, Boolean confirm, boolean logicalDelete)
+            throws PersistenceException, BusinessException {
+        if (!ERepositoryObjectType.getAllTypesOfCodesJar().contains(currentJobNode.getObjectType())) {
+            return;
+        }
+        if (!currentJobNode.getChildren().isEmpty()) {
+            List<IRepositoryViewObject> deleteObjectList = new ArrayList<IRepositoryViewObject>();
+            for (IRepositoryNode child : currentJobNode.getChildren()) {
+                deleteElements(factory, deleteActionCache, (RepositoryNode) child, confirm, deleteObjectList);
+            }
+            if (deleteObjectList != null && deleteObjectList.size() > 0) {
+                factory.batchDeleteObjectPhysical4Remote(ProjectManager.getInstance().getCurrentProject(), deleteObjectList);
+            }
+        }
+
+        // delete forever to delete codeJar folder
+        if (!logicalDelete) {
+            IFolder folder = ResourceUtils
+                    .getFolder(ResourceUtils.getProject(ProjectManager.getInstance().getCurrentProject()),
+                            ERepositoryObjectType.getFolderName(currentJobNode.getObjectType()), true)
+                    .getFolder(currentJobNode.getObject().getProperty().getLabel());
+            if (folder != null) {
+                try {
+                    folder.delete(false, null);
+                } catch (CoreException e) {
+                    throw new PersistenceException(e);
+                }
+            }
+        }
+
+    }
+
     private void deleteTestCases(IProxyRepositoryFactory factory, DeleteActionCache deleteActionCache,
             final IRepositoryNode currentJobNode, Boolean confirm, List<IRepositoryViewObject> batchDeleteObjectList)
             throws PersistenceException, BusinessException {

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/DuplicateAction.java

@@ -74,6 +74,7 @@ import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryContentHandler;
 import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.repository.RepositoryContentManager;
+import org.talend.core.model.routines.RoutinesUtil;
 import org.talend.core.repository.i18n.Messages;
 import org.talend.core.repository.model.ProxyRepositoryFactory;
 import org.talend.core.repository.ui.dialog.DuplicateDialog;
@@ -145,8 +146,14 @@ public class DuplicateAction extends AContextualAction {
                 canWork = false;
             } else if (((StructuredSelection) selection).toArray().length == 1) {
                 Object obj = ((StructuredSelection) selection).toList().get(0);
+                RepositoryNode sourceNode = (RepositoryNode) obj;
+                // disable duplicate for codejar / innercode now
+                if (ERepositoryObjectType.getAllTypesOfCodesJar().contains(sourceNode.getObjectType())
+                        || sourceNode.getObject() != null && sourceNode.getObject().getProperty() != null
+                                && RoutinesUtil.isInnerCodes(sourceNode.getObject().getProperty())) {
+                    canWork = false;
+                }
                 if (canWork) {
-                    RepositoryNode sourceNode = (RepositoryNode) obj;
                     if (!CopyObjectAction.getInstance().validateAction(sourceNode, null)) {
                         canWork = false;
                     } else if (node.getProperties(EProperties.CONTENT_TYPE) == ERepositoryObjectType.JOB_DOC

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/EmptyRecycleBinAction.java

@@ -16,6 +16,7 @@ import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.resources.IWorkspace;
 import org.eclipse.core.resources.IWorkspaceRunnable;
 import org.eclipse.core.resources.ResourcesPlugin;
@@ -45,6 +46,7 @@ import org.talend.commons.ui.runtime.exception.ExceptionHandler;
 import org.talend.commons.ui.runtime.exception.MessageBoxExceptionHandler;
 import org.talend.commons.ui.runtime.image.ECoreImage;
 import org.talend.commons.ui.runtime.image.ImageProvider;
+import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.ITDQRepositoryService;
 import org.talend.core.model.properties.ConnectionItem;
@@ -343,6 +345,7 @@ public class EmptyRecycleBinAction extends AContextualAction {
                             testService.deleteDataFiles(objToDelete);
                         }
                     }
+                    deleteCodeSubItem(factory, currentNode);
                     if (!ProjectManager.getInstance().getCurrentProject().isLocal()) {
                         // if remote, batch delete later
                         batchDeleteObjectList.add(objToDelete);
@@ -355,6 +358,36 @@ public class EmptyRecycleBinAction extends AContextualAction {
         }
     }
 
+    private void deleteCodeSubItem(IProxyRepositoryFactory factory, final IRepositoryNode currentNode)
+            throws PersistenceException, BusinessException {
+        if (!ERepositoryObjectType.getAllTypesOfCodesJar().contains(currentNode.getObjectType())) {
+            return;
+        }
+        if (!currentNode.getChildren().isEmpty()) {
+            List<IRepositoryViewObject> deleteObjectList = new ArrayList<IRepositoryViewObject>();
+            for (IRepositoryNode child : currentNode.getChildren()) {
+                deleteElements(factory, (RepositoryNode) child, deleteObjectList);
+            }
+            if (deleteObjectList != null && deleteObjectList.size() > 0) {
+                factory.batchDeleteObjectPhysical4Remote(ProjectManager.getInstance().getCurrentProject(), deleteObjectList);
+            }
+        }
+
+        // delete forever to delete codeJar folder
+        IFolder folder = ResourceUtils
+                .getFolder(ResourceUtils.getProject(ProjectManager.getInstance().getCurrentProject()),
+                        ERepositoryObjectType.getFolderName(currentNode.getObjectType()), true)
+                .getFolder(currentNode.getObject().getProperty().getLabel());
+        if (folder != null) {
+            try {
+                folder.delete(false, null);
+            } catch (CoreException e) {
+                throw new PersistenceException(e);
+            }
+        }
+
+    }
+
     protected boolean isRelation(IEditorInput editorInput, String repoNodeProjectLabel, String repoNodeId) {
         IRepoViewLinker[] allRepoViewLinkers = RepoViewLinkerRegistryReader.getInstance().getAllRepoViewLinkers();
         for (IRepoViewLinker linker : allRepoViewLinkers) {

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/actions/RestoreAction.java

@@ -193,7 +193,8 @@ public class RestoreAction extends AContextualAction {
                             node = node.getParent();
                             if ((node.getObject().getProperty().getItem() instanceof FolderItem)) {
                                 node.getObject().getProperty().getItem().getState().setDeleted(false);
-                            } else if (isTestcase) {
+                            } else if (isTestcase
+                                    || ERepositoryObjectType.getAllTypesOfCodesJar().contains(node.getObjectType())) {
                                 restoreNode(node);
                             }
                         }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/ui/view/RepositoryLabelProvider.java

@@ -389,6 +389,14 @@ public class RepositoryLabelProvider extends LabelProvider implements IColorProv
         case STABLE_SYSTEM_FOLDER:
         case SYSTEM_FOLDER:
             ERepositoryObjectType contentType = node.getContentType();
+            if (contentType != null) {
+                // special setting for system codeJar node
+                if (contentType.equals(ERepositoryObjectType.ROUTINESJAR)) {
+                    nodeIcon = RepositoryImageProvider.getIcon(ERepositoryObjectType.ROUTINES);
+                } else if (contentType.equals(ERepositoryObjectType.BEANSJAR)) {
+                    nodeIcon = RepositoryImageProvider.getIcon(ERepositoryObjectType.BEANS);
+                }
+            }
             if (nodeIcon == null || EImage.DEFAULT_IMAGE.equals(nodeIcon)) {
                 Image image = getImageFromFramework(contentType);
                 if (image != null) {

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/AbstractResourceRepositoryContentHandler.java

@@ -22,6 +22,7 @@ import org.talend.core.model.properties.ByteArray;
 import org.talend.core.model.properties.FileItem;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.ProcessItem;
+import org.talend.core.model.properties.RoutinesJarItem;
 import org.talend.core.model.repository.AbstractRepositoryContentHandler;
 import org.talend.core.model.repository.ERepositoryObjectType;
 
@@ -66,6 +67,13 @@ public abstract class AbstractResourceRepositoryContentHandler extends AbstractR
         return itemResource;
     }
 
+    protected Resource save(RoutinesJarItem item) {
+        final Resource itemResource = getXmiResourceManager().getItemResource(item);
+        itemResource.getContents().clear();
+        itemResource.getContents().add(item.getRoutinesJarType());
+        return itemResource;
+    }
+
     protected Resource saveScreenShots(ProcessItem item) throws PersistenceException {
         Resource itemResource = getXmiResourceManager().getScreenshotResource(item, true, true);
         EMap screenshots = item.getProcess().getScreenshots();
@@ -76,7 +84,7 @@ public abstract class AbstractResourceRepositoryContentHandler extends AbstractR
         return itemResource;
     }
 
-    private XmiResourceManager getXmiResourceManager() {
+    protected XmiResourceManager getXmiResourceManager() {
         if (null == xmiResourceManager) {
             xmiResourceManager = new XmiResourceManager();
         }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/ProjectDataJsonProvider.java

@@ -81,6 +81,12 @@ public class ProjectDataJsonProvider {
         return strBuilder.toString();
     }
 
+    public static String getMigrationTaskIndexPath() {
+        StringBuilder strBuilder = new StringBuilder();
+        strBuilder.append(FileConstants.SETTINGS_FOLDER_NAME).append("/").append(FileConstants.MIGRATION_TASK_FILE_NAME); //$NON-NLS-1$
+        return strBuilder.toString();
+    }
+
     public static void saveProjectData(Project project) throws PersistenceException {
         saveProjectData(project, CONTENT_ALL);
     }

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/RoutineUtils.java

@@ -12,6 +12,10 @@
 // ============================================================================
 package org.talend.core.repository.utils;
 
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
 import org.apache.oro.text.regex.MalformedPatternException;
 import org.apache.oro.text.regex.Pattern;
 import org.apache.oro.text.regex.PatternCompiler;
@@ -26,37 +30,65 @@ import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.RoutineItem;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.repository.model.ProxyRepositoryFactory;
+import org.talend.core.runtime.services.IDesignerMavenService;
 
 /**
  * DOC nrousseau class global comment. Detailled comment
  */
 public final class RoutineUtils {
 
+    private static final String DEFAULT_PACKAGE_REGEX = "package(\\s)+" + JavaUtils.JAVA_ROUTINES_DIRECTORY //$NON-NLS-1$
+            + "\\.((\\w)+)(\\s)*;"; //$NON-NLS-1$
+
+    private static final String DEFAULT_PACKAGE_STRING = "package " + JavaUtils.JAVA_ROUTINES_DIRECTORY + ";"; //$NON-NLS-1$ //$NON-NLS-2$
+
+    private static final String INNER_ROUTINES_PACKAGE_REGEX = "package\\s+([^;]+);";
+
     public static void changeRoutinesPackage(Item item) {
+        List<ERepositoryObjectType> allowedTypes = new ArrayList<ERepositoryObjectType>();
+        allowedTypes.add(ERepositoryObjectType.ROUTINES);
+        doChangeRoutinesPackage(item, DEFAULT_PACKAGE_REGEX, DEFAULT_PACKAGE_STRING, allowedTypes, false, true);
+    }
+
+    public static void changeInnerCodePackage(Item item, boolean avoidSave) {
+        changeInnerCodePackage(item, avoidSave, true);
+    }
+
+    public static void changeInnerCodePackage(Item item, boolean avoidSave, boolean commitMode) {
+        IDesignerMavenService service = IDesignerMavenService.get();
+        if (service != null && item instanceof RoutineItem) {
+            RoutineItem routineItem = (RoutineItem) item;
+            String codesJarPackageByInnerCode = service.getCodesJarPackageByInnerCode(routineItem);
+            if (StringUtils.isNotBlank(codesJarPackageByInnerCode)) {
+                String newPackageString = "package " + StringUtils.replace(codesJarPackageByInnerCode, "/", ".") + ";";
+                if (!routineItem.isBuiltIn()) {
+                    String routineContent = new String(routineItem.getContent().getInnerContent());
+                    if (routineContent != null && routineContent.contains(newPackageString)) {
+                        return;
+                    }
+                }
+                doChangeRoutinesPackage(item, INNER_ROUTINES_PACKAGE_REGEX, newPackageString,
+                        ERepositoryObjectType.getAllTypesOfCodes(), avoidSave, commitMode);
+            }
+        }
+    }
+
+    public static void doChangeRoutinesPackage(Item item, String packageRegex, String newPackage,
+            List<ERepositoryObjectType> allowedTypes, boolean avoidSave, boolean commitMode) {
         if (item == null) {
             return;
         }
 
         ERepositoryObjectType itemType = ERepositoryObjectType.getItemType(item);
-        if (ERepositoryObjectType.ROUTINES.equals(itemType) && item instanceof RoutineItem) {
+        if (allowedTypes != null && allowedTypes.contains(itemType) && item instanceof RoutineItem) {
             RoutineItem rItem = (RoutineItem) item;
             if (!rItem.isBuiltIn()) {
-                //
                 String routineContent = new String(rItem.getContent().getInnerContent());
-                //
-                // String curProjectName =
-                // currentProject.getTechnicalLabel().toLowerCase();
-                String oldPackage = "package(\\s)+" + JavaUtils.JAVA_ROUTINES_DIRECTORY + "\\.((\\w)+)(\\s)*;"; //$NON-NLS-1$ //$NON-NLS-2$
-                // String newPackage = "package " +
-                // JavaUtils.JAVA_ROUTINES_DIRECTORY + "." + curProjectName +
-                // ";";
-
-                String newPackage = "package " + JavaUtils.JAVA_ROUTINES_DIRECTORY + ";"; //$NON-NLS-1$ //$NON-NLS-2$
                 try {
                     PatternCompiler compiler = new Perl5Compiler();
                     Perl5Matcher matcher = new Perl5Matcher();
                     matcher.setMultiline(true);
-                    Pattern pattern = compiler.compile(oldPackage);
+                    Pattern pattern = compiler.compile(packageRegex);
 
                     if (matcher.contains(routineContent, pattern)) {
                         // String group = matcher.getMatch().group(2);
@@ -67,7 +99,14 @@ public final class RoutineUtils {
                         rItem.getContent().setInnerContent(routineContent.getBytes());
                         ProxyRepositoryFactory repFactory = ProxyRepositoryFactory.getInstance();
 
-                        repFactory.save(rItem);
+                        if (!avoidSave) {
+                            if (commitMode) {
+                                repFactory.save(rItem);
+                            } else {
+                                // avoid deadlock in git pull event listener
+                                new XmiResourceManager().saveResource(rItem.eResource());
+                            }
+                        }
                         // }
                     }
                 } catch (MalformedPatternException e) {

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/StandaloneConnectionContextUtils.java

@@ -205,6 +205,27 @@ public class StandaloneConnectionContextUtils {
             cloneConn.getParameters().put(ConnParameterKeys.CONN_PARA_KEY_HIVE_ADDITIONAL_JDBC_SETTINGS,
                     getOriginalValue(contextProperties, additionalJDBCSettings));
 
+            String dataprocProjectId = cloneConn.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_HIVE_GOOGLE_PROJECT_ID);
+            cloneConn.getParameters().put(ConnParameterKeys.CONN_PARA_KEY_HIVE_GOOGLE_PROJECT_ID,
+                    getOriginalValue(contextProperties, dataprocProjectId));
+
+            String dataprocClusterId = cloneConn.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_HIVE_GOOGLE_CLUSTER_ID);
+            cloneConn.getParameters().put(ConnParameterKeys.CONN_PARA_KEY_HIVE_GOOGLE_CLUSTER_ID,
+                    getOriginalValue(contextProperties, dataprocClusterId));
+
+            String dataprocRegion = cloneConn.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_HIVE_GOOGLE_REGION);
+            cloneConn.getParameters().put(ConnParameterKeys.CONN_PARA_KEY_HIVE_GOOGLE_REGION,
+                    getOriginalValue(contextProperties, dataprocRegion));
+
+            String dataprocJarsBucket = cloneConn.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_HIVE_GOOGLE_JARS_BUCKET);
+            cloneConn.getParameters().put(ConnParameterKeys.CONN_PARA_KEY_HIVE_GOOGLE_JARS_BUCKET,
+                    getOriginalValue(contextProperties, dataprocJarsBucket));
+
+            String dataprocPathToCredentials = cloneConn.getParameters()
+                    .get(ConnParameterKeys.CONN_PARA_KEY_HIVE_AUTHENTICATION_PATH_TO_GOOGLE_CREDENTIALS);
+            cloneConn.getParameters().put(ConnParameterKeys.CONN_PARA_KEY_HIVE_AUTHENTICATION_PATH_TO_GOOGLE_CREDENTIALS,
+                    getOriginalValue(contextProperties, dataprocPathToCredentials));
+
             String hiveEnableHa = cloneConn.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_HIVE_ENABLE_HA);
             cloneConn.getParameters().put(ConnParameterKeys.CONN_PARA_KEY_HIVE_ENABLE_HA,
                     getOriginalValue(contextProperties, hiveEnableHa));

main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/XmiResourceManager.java

@@ -26,6 +26,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.log4j.Priority;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.resources.IProject;
@@ -37,6 +38,7 @@ import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.Path;
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.emf.common.util.EList;
 import org.eclipse.emf.common.util.URI;
 import org.eclipse.emf.ecore.EObject;
 import org.eclipse.emf.ecore.resource.Resource;
@@ -68,7 +70,9 @@ import org.talend.core.model.properties.TDQItem;
 import org.talend.core.model.properties.ValidationRulesConnectionItem;
 import org.talend.core.model.properties.helper.ByteArrayResource;
 import org.talend.core.model.repository.ERepositoryObjectType;
+import org.talend.core.model.routines.RoutinesUtil;
 import org.talend.core.repository.constants.FileConstants;
+import org.talend.core.repository.model.ProxyRepositoryFactory;
 import org.talend.core.repository.utils.ResourceFilenameHelper.FileName;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
@@ -206,7 +210,7 @@ public class XmiResourceManager {
      * @return
      * @throws PersistenceException
      */
-    private IPath getFolderPath(IProject project, ERepositoryObjectType repositoryObjectType, IPath relativePath)
+    private IPath getFolderPath(IProject project, ERepositoryObjectType repositoryObjectType, IPath relativePath, Item item)
             throws PersistenceException {
         ERepositoryObjectType type = repositoryObjectType;
         if (ERepositoryObjectType.TDQ_SYSTEM_INDICATORS.equals(repositoryObjectType)
@@ -215,6 +219,10 @@ public class XmiResourceManager {
         } else if (ERepositoryObjectType.TDQ_PATTERN_REGEX.equals(repositoryObjectType)
                 || ERepositoryObjectType.TDQ_PATTERN_SQL.equals(repositoryObjectType)) {
             type = ERepositoryObjectType.TDQ_PATTERN_ELEMENT;
+        } else if (type == ERepositoryObjectType.ROUTINES && RoutinesUtil.isInnerCodes(item.getProperty())) {
+            type = ERepositoryObjectType.ROUTINESJAR;
+        } else if (type == ERepositoryObjectType.BEANS && RoutinesUtil.isInnerCodes(item.getProperty())) {
+            type = ERepositoryObjectType.BEANSJAR;
         }
         IFolder folder = project.getFolder(ERepositoryObjectType.getFolderName(type)).getFolder(relativePath);
         return folder.getFullPath();
@@ -647,6 +655,23 @@ public class XmiResourceManager {
     }
 
     public void saveResource(Resource resource) throws PersistenceException {
+        try {
+            if (resource != null) {
+                Object objectByType = EcoreUtil.getObjectByType(resource.getContents(), PropertiesPackage.eINSTANCE.getProject());
+                if (objectByType != null) {
+                    Project project = (Project) objectByType;
+                    EList migrationTasks = project.getMigrationTask();
+                    if (migrationTasks != null && 1 < migrationTasks.size()) {
+                        org.talend.commons.exception.ExceptionHandler.process(new Exception("Bad saving logic for Project"),
+                                Priority.WARN);
+                        ProxyRepositoryFactory.getInstance().saveProject(new org.talend.core.model.general.Project(project));
+                        return;
+                    }
+                }
+            }
+        } catch (Throwable e) {
+            org.talend.commons.exception.ExceptionHandler.process(e);
+        }
         EmfHelper.saveResource(resource);
     }
 
@@ -684,7 +709,7 @@ public class XmiResourceManager {
     // MOD mzhao 2010-11-22, suppport TDQ item file extensions.(.ana, .rep, etc)
     private URI getItemResourceURI(IProject project, ERepositoryObjectType repositoryObjectType, IPath path, Item item)
             throws PersistenceException {
-        IPath folderPath = getFolderPath(project, repositoryObjectType, path);
+        IPath folderPath = getFolderPath(project, repositoryObjectType, path, item);
         FileName fileName = ResourceFilenameHelper.create(item.getProperty());
         IPath resourcePath = null;
         if (item.getFileExtension() == null) {
@@ -700,7 +725,7 @@ public class XmiResourceManager {
     @Deprecated
     private URI getItemResourceURI(IProject project, ERepositoryObjectType repositoryObjectType, IPath path, Item item,
             String fileExtension) throws PersistenceException {
-        IPath folderPath = getFolderPath(project, repositoryObjectType, path);
+        IPath folderPath = getFolderPath(project, repositoryObjectType, path, item);
         FileName fileName = ResourceFilenameHelper.create(item.getProperty());
         IPath resourcePath = ResourceFilenameHelper
                 .getExpectedFilePath(fileName, folderPath, fileExtension, item.isNeedVersion());
@@ -710,7 +735,7 @@ public class XmiResourceManager {
     // added by dlin 2011-7-14 to create the uri of file of .screenshot
     private URI getScreenshotResourceURI(IProject project, ERepositoryObjectType repositoryObjectType, IPath path, Item item,
             String... fileExtension) throws PersistenceException {
-        IPath folderPath = getFolderPath(project, repositoryObjectType, path);
+        IPath folderPath = getFolderPath(project, repositoryObjectType, path, item);
         FileName fileName = ResourceFilenameHelper.create(item.getProperty());
         IPath resourcePath = ResourceFilenameHelper.getExpectedFilePath(fileName, folderPath, FileConstants.SCREENSHOT_EXTENSION,
                 item.isNeedVersion());

main/plugins/org.talend.core.runtime/META-INF/MANIFEST.MF

@@ -95,6 +95,7 @@ Export-Package: org.talend.commons.utils.generation,
  org.talend.repository.documentation.generation,
  org.talend.repository.model,
  org.talend.repository.model.nodes,
+ org.talend.repository.token,
  org.talend.repository.ui.actions,
  org.talend.repository.ui.dialog,
  org.talend.repository.ui.login.connections,
@@ -122,7 +123,8 @@ Require-Bundle: org.eclipse.jdt.core,
  org.talend.components.api,
  org.apache.servicemix.bundles.avro,
  jackson-core-asl,
- org.talend.libraries.jackson
+ org.talend.libraries.jackson,
+ org.talend.libraries.apache.common
 Bundle-Activator: org.talend.core.runtime.CoreRuntimePlugin
 Bundle-ActivationPolicy: lazy
 Bundle-ClassPath: .,

main/plugins/org.talend.core.runtime/mappings/mapping_Databricks_Delta_Lake.xml

@@ -0,0 +1,128 @@
+<?xml version="1.0"?>
+<mapping>
+	<dbms product="DATABRICKS_DELTA_LAKE" id="databricks_delta_lake_id" label="Mapping Delta Lake" default="true">
+		<dbTypes>
+			<dbType type="SMALLINT" ignoreLen="true" ignorePre="true"/>
+			<dbType type="FLOAT" ignoreLen="true" ignorePre="true"/>
+			<dbType type="DOUBLE" ignoreLen="true" ignorePre="true"/>
+			<dbType type="BIGINT" ignoreLen="true" ignorePre="true"/>
+			<dbType type="INT" ignoreLen="true" ignorePre="true"/>
+			<dbType type="TINYINT" ignoreLen="true" ignorePre="true"/>
+			<dbType type="STRING" ignoreLen="true" ignorePre="true"/>
+			<dbType type="DECIMAL" ignoreLen="false" ignorePre="false"/>
+			<dbType type="BOOLEAN" ignoreLen="true" ignorePre="true"/>
+			<dbType type="STRUCT" ignoreLen="true" ignorePre="true"/>
+			<dbType type="MAP" ignoreLen="true" ignorePre="true"/>
+			<dbType type="ARRAY" ignoreLen="true" ignorePre="true"/>
+			<dbType type="TIMESTAMP" ignoreLen="true" ignorePre="true"/>
+			<dbType type="DATE" ignoreLen="true" ignorePre="true"/>
+		</dbTypes>
+		
+		<language name="java">
+			<talendToDbTypes><!-- Adviced mappings  -->
+				<talendType type="id_List"/>
+				<talendType type="id_Boolean">	
+					<dbType type="BOOLEAN" default="true"/>				
+				</talendType>
+				<talendType type="id_Byte">
+					<dbType type="TINYINT" default="true"/>
+					<dbType type="BIGINT"/>
+					<dbType type="INT"/>
+					<dbType type="SMALLINT"/>				
+				</talendType>
+				<talendType type="id_byte[]"/>
+				<talendType type="id_Character">
+					<dbType type="STRING" default="true"/>
+				</talendType>
+				<talendType type="id_Date">
+				    <dbType type="TIMESTAMP" default="true"/>
+				    <dbType type="DATE"/>
+				</talendType>
+				<talendType type="id_BigDecimal">
+				    <dbType type="DECIMAL" default="true"/>
+					<dbType type="BIGINT"/>
+					<dbType type="FLOAT"/>
+					<dbType type="DOUBLE"/>
+				</talendType>
+				<talendType type="id_Double">
+					<dbType type="DOUBLE" default="true" />
+					<dbType type="FLOAT"/>
+				</talendType>
+				<talendType type="id_Float">
+					<dbType type="FLOAT" default="true" />
+					<dbType type="DOUBLE"/>
+				</talendType>
+				<talendType type="id_Integer">
+					<dbType type="INT" default="true" />
+					<dbType type="BIGINT" />
+				</talendType>
+				<talendType type="id_Long">
+					<dbType type="BIGINT" default="true" />		
+				</talendType>
+				<talendType type="id_Object">
+					<dbType type="STRUCT" default="true" />
+					<dbType type="MAP" />
+					<dbType type="ARRAY" />	
+				</talendType>
+				<talendType type="id_Short">
+					<dbType type="SMALLINT" default="true" />
+					<dbType type="INT" />
+				</talendType>
+				<talendType type="id_String">
+					<dbType type="STRING" default="true"/>
+				</talendType>
+			</talendToDbTypes>
+			<dbToTalendTypes><!-- Adviced mappings  -->
+				<dbType type="STRING">
+					<talendType type="id_String" default="true"/>
+				</dbType>
+				<dbType type="BOOLEAN">
+					<talendType type="id_Boolean" default="true"/>
+				</dbType>
+				<dbType type="DOUBLE">
+					<talendType type="id_Double" default="true"/>
+					<talendType type="id_BigDecimal"/>
+					<talendType type="id_Float"/>
+				</dbType>
+                <dbType type="DECIMAL">
+                    <talendType type="id_BigDecimal" default="true"/>
+                </dbType>
+				<dbType type="BIGINT">
+					<talendType type="id_Long" default="true"/>
+					<talendType type="id_Integer"/>
+				</dbType>
+				<dbType type="INT">
+					<talendType type="id_Integer" default="true"/>
+					<talendType type="id_Short"/>
+				</dbType>
+				<dbType type="SMALLINT">
+					<talendType type="id_Short" default="true"/>
+					<talendType type="id_Byte"/>
+				</dbType>
+				<dbType type="TINYINT">
+					<talendType type="id_Byte" default="true"/>
+				</dbType>
+				<dbType type="FLOAT">
+					<talendType type="id_Float" default="true"/>
+					<talendType type="id_Double"/>
+					<talendType type="id_BigDecimal"/>
+				</dbType>
+				<dbType type="STRUCT">
+					<talendType type="id_Object" default="true"/>
+				</dbType>
+				<dbType type="MAP">
+					<talendType type="id_Object" default="true"/>
+				</dbType>
+				<dbType type="ARRAY">
+					<talendType type="id_Object" default="true"/>
+				</dbType>
+                <dbType type="TIMESTAMP">
+                    <talendType type="id_Date" default="true"/>
+                </dbType>
+                <dbType type="DATE">
+                    <talendType type="id_Date" default="true"/>
+                </dbType>
+			</dbToTalendTypes>
+		</language>
+	</dbms>
+</mapping>

main/plugins/org.talend.core.runtime/mappings/mapping_Greenplum.xml

@@ -12,17 +12,26 @@
 			<dbType type="BPCHAR" ignoreLen="true" ignorePre="true" />
 			<dbType type="BYTEA" ignoreLen="true" ignorePre="true" />	
 			<dbType type="CHAR" defaultLength="50" ignorePre="true"/>
+			<dbType type="CHARACTER" defaultLength="50" ignorePre="true"/>
 			<dbType type="CID" ignoreLen="true" ignorePre="true" />
 			<dbType type="CIDR" ignoreLen="true" ignorePre="true" />
 			<dbType type="CIRCLE" ignoreLen="true" ignorePre="true" />
 			<dbType type="DATE" ignoreLen="true" ignorePre="true" />
 			<dbType type="FLOAT4" ignoreLen="true" ignorePre="true" />
+			<dbType type="REAL" ignoreLen="true" ignorePre="true" />
 			<dbType type="FLOAT8" ignoreLen="true" ignorePre="true" />
+			<dbType type="DOUBLE PRECISION" ignoreLen="true" ignorePre="true" />
 			<dbType type="INET" ignoreLen="true" ignorePre="true" />
 			<dbType type="INT2" ignoreLen="true" ignorePre="true" />
+			<dbType type="SMALLINT" ignoreLen="true" ignorePre="true" />
 			<dbType type="INT2VECTOR" ignoreLen="true" ignorePre="true" />
 			<dbType type="INT4" ignoreLen="true" ignorePre="true" />
 			<dbType type="INT8" ignoreLen="true" ignorePre="true" />
+			<dbType type="INTEGER" ignoreLen="true" ignorePre="true" />
+			<dbType type="CHARACTER VARYING" ignoreLen="true" ignorePre="true" />
+			<dbType type="BOOLEAN" ignoreLen="true" ignorePre="true" />
+			<dbType type="BIGINT" ignoreLen="true" ignorePre="true" />
+			<dbType type="BIT VARYING" ignoreLen="true" ignorePre="true" />
 			<dbType type="INTERVAL" ignorePre="true" />
 			<dbType type="LINE" ignoreLen="true" ignorePre="true" />
 			<dbType type="LSEG" ignoreLen="true" ignorePre="true" />
@@ -30,6 +39,7 @@
 			<dbType type="MONEY" ignoreLen="true" ignorePre="true" />	
 			<dbType type="NAME" ignoreLen="true" ignorePre="true" />
 			<dbType type="NUMERIC" defaultLength="20" defaultPrecision="10"/>
+			<dbType type="DECIMAL" defaultLength="20" defaultPrecision="10"/>
 			<dbType type="OID" ignoreLen="true" ignorePre="true"  />
 			<dbType type="OIDVECTOR" ignoreLen="true" ignorePre="true" />
 			<dbType type="PATH" ignoreLen="true" ignorePre="true" />
@@ -110,16 +120,19 @@
 			<talendToDbTypes><!-- Adviced mappings  -->
 				<talendType type="id_List"/>
 				<talendType type="id_Boolean">	
-					<dbType type="BOOL" default="true" />				
+					<dbType type="BOOL" default="true" />
+					<dbType type="BOOLEAN" />					
 				</talendType>
 				<talendType type="id_Byte">		
 				</talendType>
 				<talendType type="id_byte[]">
 					<dbType type="BIT" default="true" />
-					<dbType type="VARBIT" />				
+					<dbType type="VARBIT" />
+					<dbType type="BIT VARYING" />				
 				</talendType>
 				<talendType type="id_Character">
 					<dbType type="CHAR" default="true" />
+					<dbType type="CHARACTER" />
 					<dbType type="BPCHAR" />
 					<dbType type="VARCHAR" />
 					<dbType type="TEXT" />							
@@ -136,33 +149,47 @@
 					<dbType type="RELTIME" />						
 				</talendType>
 				<talendType type="id_BigDecimal">
-					<dbType type="NUMERIC" default="true" />							
+					<dbType type="NUMERIC" default="true" />
+					<dbType type="DECIMAL" />							
 					<dbType type="FLOAT4" />
+					<dbType type="REAL" />
 					<dbType type="FLOAT8"/>
+					<dbType type="DOUBLE PRECISION"/>
 				</talendType>
 				<talendType type="id_Double">
 					<dbType type="FLOAT4" />
+					<dbType type="REAL" />
 					<dbType type="FLOAT8" default="true" />
-					<dbType type="NUMERIC" />							
+					<dbType type="NUMERIC" />
+					<dbType type="DECIMAL" />	
+					<dbType type="DOUBLE PRECISION"/>						
 				</talendType>
 				<talendType type="id_Float">
 					<dbType type="FLOAT4" default="true" />
 					<dbType type="FLOAT8" />
-					<dbType type="NUMERIC" />				
+					<dbType type="REAL" />
+					<dbType type="NUMERIC" />
+					<dbType type="DECIMAL" />
+					<dbType type="DOUBLE PRECISION"/>				
 				</talendType>
 				<talendType type="id_Integer">
 					<dbType type="INT2" />
+					<dbType type="SMALLINT" />
 					<dbType type="INT4" default="true" />
 					<dbType type="SERIAL" />
 					<dbType type="INT8" />
-					<dbType type="BIGSERIAL" />										
+					<dbType type="BIGINT" />
+					<dbType type="BIGSERIAL" />
+					<dbType type="INTEGER" />										
 				</talendType>
 				<talendType type="id_Long">
 					<dbType type="INT2" />
 					<dbType type="INT4" />
 					<dbType type="SERIAL" />
 					<dbType type="INT8"  default="true" />
-					<dbType type="BIGSERIAL" />							
+					<dbType type="BIGINT" />
+					<dbType type="BIGSERIAL" />	
+					<dbType type="SMALLINT" />						
 				</talendType>
 				<talendType type="id_Object">
 					<dbType type="_POINT" />
@@ -209,18 +236,26 @@
 					<dbType type="_ABSTIME" />
 					<dbType type="_ACLITEM" />
 					<dbType type="_INT8" />
+					<dbType type="BIGINT" />
+					<dbType type="BIT VARYING" />
 					<dbType type="_VARBIT" />
 					<dbType type="_BIT" />			
 					<dbType type="_BOOL" />
+					<dbType type="BOOLEAN" />
 					<dbType type="BOX" />
 					<dbType type="_BOX" />
 					<dbType type="_BYTEA" />
 					<dbType type="_VARCHAR" />
+					<dbType type="CHARACTER VARYING" />
+					<dbType type="CHARACTER" />
+					<dbType type="INTEGER" />
+					<dbType type="REAL" />
 					<dbType type="_BPCHAR" />
 					<dbType type="_CID" />
 					<dbType type="_CIDR" />
 					<dbType type="_CIRCLE" />
 					<dbType type="_DATE" />
+					<dbType type="DOUBLE PRECISION"/>
 					<dbType type="_FLOAT8" />
 					<dbType type="_INET" />
 					<dbType type="INT2VECTOR" />
@@ -236,6 +271,7 @@
 					<dbType type="NAME" />
 					<dbType type="_NAME" />
 					<dbType type="_NUMERIC" />
+					<dbType type="DECIMAL" />
 					<dbType type="_OID" />
 					<dbType type="OIDVECTOR" />
 					<dbType type="_OIDVECTOR" />
@@ -247,26 +283,44 @@
 					<dbType type="INT4" />
 					<dbType type="SERIAL" />
 					<dbType type="INT8" />
-					<dbType type="BIGSERIAL" />					
+					<dbType type="BIGINT" />
+					<dbType type="BIGSERIAL" />
+					<dbType type="SMALLINT" />					
 				</talendType>
 				<talendType type="id_String">
 					<dbType type="BPCHAR" />
 					<dbType type="VARCHAR" default="true" />
+					<dbType type="CHARACTER VARYING" />
 					<dbType type="TEXT" />				
 				</talendType>
 			</talendToDbTypes>
 			<dbToTalendTypes>
 				<dbType type="INT2" >
 					<talendType type="id_Short" default="true" />				
+				</dbType>
+				<dbType type="SMALLINT" >
+					<talendType type="id_Short" default="true" />				
 				</dbType>				
 				<dbType type="INT4" >
 					<talendType type="id_Integer" default="true" />
-				</dbType>				
+				</dbType>
+				<dbType type="INTEGER" >
+     				<talendType type="id_Integer" default="true" />
+    			</dbType>
+    			<dbType type="CHARACTER VARYING" >
+					<talendType type="id_String" default="true" />				
+				</dbType>
+				<dbType type="BOOLEAN" >
+					<talendType type="id_Boolean" default="true" />				
+				</dbType> 				
 				<dbType type="SERIAL" >
 					<talendType type="id_Integer" default="true" />				
 				</dbType>				
 				<dbType type="INT8" >
 					<talendType type="id_Long" default="true" />				
+				</dbType>
+				<dbType type="BIGINT" >
+					<talendType type="id_Long" default="true" />				
 				</dbType>				
 				<dbType type="BIGSERIAL" >
 					<talendType type="id_Long" default="true" />				
@@ -274,14 +328,26 @@
 				<dbType type="FLOAT4" >
 					<talendType type="id_Float" default="true" />				
 					<talendType type="id_BigDecimal"/>			
-				</dbType>				
+				</dbType>
+				<dbType type="REAL" >
+					<talendType type="id_Float" default="true" />				
+					<talendType type="id_BigDecimal"/>			
+				</dbType>			
 				<dbType type="FLOAT8" >
 					<talendType type="id_Double" default="true" />				
 					<talendType type="id_BigDecimal"/>			
 				</dbType>
+				<dbType type="DOUBLE PRECISION" >
+					<talendType type="id_Double" default="true" />				
+					<talendType type="id_BigDecimal"/>			
+				</dbType>
 				<dbType type="NUMERIC">
 					<talendType type="id_Double"/>
 					<talendType type="id_BigDecimal" default="true" />			
+				</dbType>
+				<dbType type="DECIMAL">
+					<talendType type="id_Double"/>
+					<talendType type="id_BigDecimal" default="true" />			
 				</dbType>				
 				<dbType type="BOOL" >
 					<talendType type="id_Boolean" default="true" />				
@@ -291,9 +357,15 @@
 				</dbType>				
 				<dbType type="VARBIT" >
 					<talendType type="id_byte[]" default="true" />				
-				</dbType>							
+				</dbType>	
+				<dbType type="BIT VARYING" >
+					<talendType type="id_byte[]" default="true" />				
+				</dbType>						
 				<dbType type="CHAR" >
 					<talendType type="id_String" default="true" />				
+				</dbType>
+				<dbType type="CHARACTER" >
+					<talendType type="id_String" default="true" />				
 				</dbType>				
 				<dbType type="BPCHAR" >
 					<talendType type="id_String" default="true" />				

main/plugins/org.talend.core.runtime/mappings/mapping_SingleStore.xml

@@ -0,0 +1,288 @@
+<?xml version="1.0"?>
+<mapping>
+	<dbms product="SINGLESTORE" id="singlestore_id" label="Mapping SingleStore"
+		default="true">
+		<dbTypes>
+			<dbType type="BOOL" ignoreLen="true" ignorePre="true" />
+			<dbType type="BOOLEAN" ignoreLen="true" ignorePre="true" />
+			<dbType type="BIGINT" ignorePre="true"/>
+			<dbType type="BIGINT UNSIGNED" ignoreLen="true" ignorePre="true" />
+			<dbType type="BINARY" ignorePre="true"/>
+			<dbType type="BIT" ignorePre="true" />
+			<dbType type="BLOB" ignoreLen="true" ignorePre="true" />
+			<dbType type="CHAR" defaultLength="200" ignorePre="true"/>
+			<dbType type="DATE" ignoreLen="true" ignorePre="true" />
+			<dbType type="DATETIME" ignoreLen="true" ignorePre="false" />
+			<dbType type="DECIMAL" defaultLength="20" defaultPrecision="10" preBeforelen="false"/>
+			<dbType type="DOUBLE" defaultLength="20" defaultPrecision="10"/>
+			<dbType type="DOUBLE UNSIGNED" ignoreLen="true" ignorePre="true" />
+			<dbType type="ENUM" ignoreLen="true" ignorePre="true" />
+			<dbType type="FLOAT" defaultPrecision="2"/>
+			<dbType type="FLOAT UNSIGNED" ignoreLen="true" ignorePre="true" />
+			<dbType type="GEOGRAPHY" ignoreLen="true" ignorePre="true" />
+			<dbType type="GEOGRAPHYPOINT" ignoreLen="true" ignorePre="true" />
+			<dbType type="INT" ignorePre="true" />
+			<dbType type="INT UNSIGNED" ignoreLen="true" ignorePre="true" />
+			<dbType type="INTEGER" ignorePre="true" />
+			<dbType type="INTEGER UNSIGNED" ignoreLen="true" ignorePre="true" />
+			<dbType type="LONGTEXT" ignoreLen="true" ignorePre="true" />
+			<dbType type="LONGBLOB" ignoreLen="true" ignorePre="true"/>
+			<dbType type="MEDIUMBLOB" ignoreLen="true" ignorePre="true" />
+			<dbType type="MEDIUMINT" ignorePre="true" />
+			<dbType type="MEDIUMINT UNSIGNED" ignoreLen="true" ignorePre="true" />
+			<dbType type="MEDIUMTEXT" ignoreLen="true" ignorePre="true" />
+			<dbType type="SMALLINT" ignorePre="true" />
+			<dbType type="SMALLINT UNSIGNED" ignoreLen="true" ignorePre="true" />
+			<dbType type="SET" ignoreLen="true" ignorePre="true" />
+			<dbType type="TEXT" ignoreLen="true" ignorePre="true" />
+			<dbType type="TIME" ignoreLen="true" ignorePre="true" />
+			<dbType type="TIMESTAMP" ignoreLen="true" ignorePre="false" />
+			<dbType type="TINYBLOB" ignoreLen="true" ignorePre="true" />
+			<dbType type="TINYINT" ignorePre="true" />
+			<dbType type="TINYINT UNSIGNED" ignoreLen="true" ignorePre="true" />
+			<dbType type="TINYTEXT" ignoreLen="true" ignorePre="true" />
+			<dbType type="VARBINARY" ignorePre="true" />
+			<dbType type="VARCHAR" default="true" defaultLength="100" ignorePre="true"/>
+			<dbType type="YEAR" ignorePre="true"/>
+			<dbType type="JSON" ignoreLen="true" ignorePre="true"/>
+		</dbTypes>
+
+		<language name="java">
+			<talendToDbTypes><!-- Adviced mappings  -->
+				<talendType type="id_List"/>
+				<talendType type="id_Boolean">	
+					<dbType type="BOOL" default="true" />				
+				</talendType>
+				<talendType type="id_Byte">
+					<dbType type="TINYINT" default="true" />
+					<dbType type="BIGINT" />
+					<dbType type="INT" />
+					<dbType type="MEDIUMINT" />
+					<dbType type="SMALLINT" />				
+				</talendType>
+				<talendType type="id_byte[]">
+				    <dbType type="VARBINARY" default="true" />
+				    <dbType type="BINARY" />
+				    <dbType type="BIT" />
+				    <dbType type="BLOB" />
+				    <dbType type="LONGBLOB" />
+				    <dbType type="MEDIUMBLOB" />
+				    <dbType type="TINYBLOB" />
+				</talendType>
+				<talendType type="id_Character">
+					<dbType type="CHAR" default="true" />
+					<dbType type="VARCHAR"/>
+				</talendType>
+				<talendType type="id_Date">
+					<dbType type="DATE" />
+					<dbType type="DATETIME" default="true" />	
+					<dbType type="TIME" />
+					<dbType type="YEAR" />
+					<dbType type="TIMESTAMP" />				
+				</talendType>
+				<talendType type="id_BigDecimal">
+					<dbType type="DECIMAL" default="true" />
+					<dbType type="FLOAT"/>
+					<dbType type="DOUBLE" />				
+				</talendType>
+				<talendType type="id_Double">
+					<dbType type="DOUBLE" default="true" />
+					<dbType type="FLOAT"/>
+					<dbType type="DECIMAL" />				
+				</talendType>
+				<talendType type="id_Float">
+					<dbType type="FLOAT" default="true" />
+					<dbType type="DOUBLE"/>
+					<dbType type="DECIMAL" />
+				</talendType>
+				<talendType type="id_Integer">
+					<dbType type="INT" default="true" />
+					<dbType type="BIGINT" />
+				</talendType>
+				<talendType type="id_Long">
+					<dbType type="BIGINT" default="true" />		
+				</talendType>
+				<talendType type="id_Object">
+					<dbType type="BLOB" default="true"/>
+					<dbType type="ENUM" />
+					<dbType type="GEOGRAPHY" />
+					<dbType type="GEOGRAPHYPOINT" />
+					<dbType type="MEDIUMINT" />
+					<dbType type="LONGBLOB" />
+					<dbType type="MEDIUMBLOB" />
+					<dbType type="SET" />
+					<dbType type="TINYBLOB" />
+				</talendType>
+				<talendType type="id_Short">
+					<dbType type="SMALLINT" default="true" />
+					<dbType type="INT" />
+					<dbType type="BIGINT"/>
+					<dbType type="MEDIUMINT" />
+				</talendType>
+				<talendType type="id_String">
+					<dbType type="VARCHAR" default="true" />
+					<dbType type="LONGTEXT"/>
+					<dbType type="MEDIUMTEXT" />
+					<dbType type="TEXT" />
+					<dbType type="TINYTEXT" />
+					<dbType type="CHAR" />
+					<dbType type="JSON" />
+				</talendType>
+			</talendToDbTypes>
+			<dbToTalendTypes>
+				<dbType type="BOOL">
+					<talendType type="id_Boolean" default="true" />
+				</dbType>
+				<dbType type="BOOLEAN">
+					<talendType type="id_Boolean" default="true" />
+				</dbType>
+				<dbType type="BIGINT">
+					<talendType type="id_Long" default="true" />
+				</dbType>
+				<dbType type="BINARY">
+					<talendType type="id_byte[]" />
+					<talendType type="id_Boolean" default="true" />
+				</dbType>
+				<dbType type="BIT">
+					<talendType type="id_byte[]" />
+					<talendType type="id_Boolean" default="true" />
+				</dbType>
+				<dbType type="BLOB">
+					<talendType type="id_byte[]" default="true" />
+					<talendType type="id_Object" />
+				</dbType>
+				<dbType type="CHAR">
+					<talendType type="id_String" default="true" />
+				</dbType>
+				<dbType type="DATE">
+					<talendType type="id_Date" default="true" />
+				</dbType>
+				<dbType type="DATETIME">
+					<talendType type="id_Date" default="true" />
+				</dbType>
+				<dbType type="DECIMAL">
+					<talendType type="id_Float"/>
+					<talendType type="id_Double"/>
+					<talendType type="id_BigDecimal" default="true"/>
+				</dbType>
+				<dbType type="DOUBLE">
+					<talendType type="id_Double" default="true" />
+					<talendType type="id_BigDecimal"/>
+				</dbType>
+				<dbType type="ENUM">
+					<talendType type="id_Object" default="true" />
+				</dbType>
+				<dbType type="FLOAT">
+					<talendType type="id_Float" default="true" />
+					<talendType type="id_Double"/>
+					<talendType type="id_BigDecimal"/>
+				</dbType>
+				<dbType type="GEOGRAPHY">
+					<talendType type="id_Object" default="true" />
+				</dbType>
+				<dbType type="GEOGRAPHYPOINT">
+					<talendType type="id_Object" default="true" />
+				</dbType>
+				<dbType type="JSON">
+					<talendType type="id_String" default="true" />
+					<talendType type="id_Object" />
+				</dbType>
+				<dbType type="INT">
+					<talendType type="id_Integer" default="true" />
+					<talendType type="id_Long"/>
+				</dbType>
+				<dbType type="INTEGER">
+					<talendType type="id_Integer" default="true" />
+					<talendType type="id_Long"/>
+				</dbType>
+				<dbType type="LONGTEXT">
+					<talendType type="id_String" default="true" />
+				</dbType>
+				<dbType type="LONGBLOB">
+					<talendType type="id_byte[]" default="true" />
+					<talendType type="id_Object" />
+				</dbType>
+				<dbType type="MEDIUMBLOB">
+					<talendType type="id_byte[]" default="true" />
+					<talendType type="id_Object" />
+				</dbType>
+				<dbType type="MEDIUMINT">
+					<talendType type="id_Integer" default="true" />
+					<talendType type="id_Long"/>
+				</dbType>
+				<dbType type="MEDIUMTEXT">
+					<talendType type="id_String" default="true" />
+				</dbType>
+				<dbType type="SMALLINT">
+					<talendType type="id_Short" default="true" />
+					<talendType type="id_Long"/>
+					<talendType type="id_Integer"/>
+				</dbType>
+				<dbType type="SET">
+					<talendType type="id_Object" default="true" />
+				</dbType>
+				<dbType type="TEXT">
+					<talendType type="id_String" default="true" />
+				</dbType>
+				<dbType type="TIME">
+					<talendType type="id_Date" default="true" />
+				</dbType>
+				<dbType type="TIMESTAMP">
+					<talendType type="id_Date" default="true" />
+				</dbType>
+				<dbType type="TINYBLOB">
+					<talendType type="id_byte[]" default="true" />
+					<talendType type="id_Object" />
+				</dbType>
+				<dbType type="TINYINT">
+					<talendType type="id_Byte" default="true" />
+					<talendType type="id_Integer"/>
+					<talendType type="id_Long"/>
+					<talendType type="id_Short"/>
+				</dbType>
+				<dbType type="TINYTEXT">
+					<talendType type="id_String" default="true" />
+				</dbType>
+				<dbType type="VARBINARY">
+				    <talendType type="id_byte[]" default="true" />
+				</dbType>
+				<dbType type="VARCHAR">
+					<talendType type="id_String" default="true" />
+				</dbType>
+				<dbType type="YEAR">
+					<talendType type="id_Date" default="true" />
+				</dbType>		
+				<dbType type="BIGINT UNSIGNED" >
+				</dbType>
+				<dbType type="DOUBLE UNSIGNED" >		
+					<talendType type="id_Double" default="true" />
+					<talendType type="id_BigDecimal"/>
+				</dbType>
+				<dbType type="FLOAT UNSIGNED" >
+					<talendType type="id_Double" default="true" />
+					<talendType type="id_BigDecimal"/>
+				</dbType>
+				<dbType type="INT UNSIGNED" >
+					<talendType type="id_Long" default="true" />
+				</dbType>
+				<dbType type="INTEGER UNSIGNED" >
+					<talendType type="id_Long" default="true" />
+				</dbType>
+				<dbType type="MEDIUMINT UNSIGNED" >
+					<talendType type="id_Integer" default="true" />
+					<talendType type="id_Long" />
+				</dbType>
+				<dbType type="SMALLINT UNSIGNED" >
+					<talendType type="id_Integer" default="true" />
+					<talendType type="id_Long" />
+				</dbType>
+				<dbType type="TINYINT UNSIGNED" >
+					<talendType type="id_Short" default="true" />
+					<talendType type="id_Integer" />
+					<talendType type="id_Long" />
+				</dbType>
+			</dbToTalendTypes>
+		</language>
+	</dbms>
+
+</mapping>

main/plugins/org.talend.core.runtime/src/main/java/org/talend/commons/utils/generation/JavaUtils.java

@@ -89,6 +89,10 @@ public final class JavaUtils {
     /** Java Routines Directory. */
     public static final String JAVA_ROUTINES_DIRECTORY = "routines"; //$NON-NLS-1$
 
+    // TODO check refrerences of JAVA_ROUTINES_DIRECTORY
+    /** Java Routines Jar Directory. */
+    public static final String JAVA_ROUTINESJAR_DIRECTORY = "routinesjar"; //$NON-NLS-1$
+
     /** Java Pig Directory. */
     public static final String JAVA_PIG_DIRECTORY = "pig"; //$NON-NLS-1$
 
@@ -98,6 +102,9 @@ public final class JavaUtils {
     /** Java Beans Directory. */
     public static final String JAVA_BEANS_DIRECTORY = "beans"; //$NON-NLS-1$
 
+    /** Java Beans Jar Directory. */
+    public static final String JAVA_BEANSJAR_DIRECTORY = "beansjar"; //$NON-NLS-1$
+
     /** Java SQLTemplate Directory. */
     public static final String JAVA_SQLPATTERNS_DIRECTORY = "sqltemplates"; //$NON-NLS-1$
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/commons/utils/io/FilesUtils.java

@@ -42,6 +42,7 @@ import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 
@@ -50,11 +51,16 @@ import org.eclipse.core.resources.IContainer;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.resources.IResource;
+import org.eclipse.core.resources.IWorkspace;
+import org.eclipse.core.resources.IWorkspaceRunnable;
+import org.eclipse.core.resources.ResourcesPlugin;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.OperationCanceledException;
+import org.eclipse.core.runtime.jobs.ISchedulingRule;
+import org.eclipse.core.runtime.jobs.Job;
 import org.osgi.framework.Bundle;
 import org.talend.commons.exception.CommonExceptionHandler;
 import org.talend.commons.exception.ExceptionHandler;
@@ -1173,4 +1179,33 @@ public class FilesUtils {
         }
     }
 
+    public static void executeFolderAction(IProgressMonitor monitor, IResource parentFolder, IWorkspaceRunnable run)
+            throws CoreException {
+        if (Job.getJobManager().currentRule() == null) {
+            IWorkspace workspace = ResourcesPlugin.getWorkspace();
+            ISchedulingRule defaultRule = workspace.getRuleFactory().modifyRule(parentFolder);
+            ISchedulingRule noBlockRule = new ISchedulingRule() {
+
+                @Override
+                public boolean isConflicting(ISchedulingRule rule) {
+                    return this.contains(rule);
+                }
+
+                @Override
+                public boolean contains(ISchedulingRule rule) {
+                    if (this.equals(rule)) {
+                        return true;
+                    }
+                    if (defaultRule.contains(rule)) {
+                        return true;
+                    }
+                    return false;
+                }
+            };
+            workspace.run(run, noBlockRule, IWorkspace.AVOID_UPDATE, monitor);
+        } else {
+            run.run(monitor);
+        }
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/IESBService.java

@@ -94,4 +94,6 @@ public interface IESBService extends IService {
      * @return
      */
     public IProcessor createOSGIJavaProcessor(IProcess process, Property property, boolean filenameFromLabel);
+    
+    public boolean isSOAPServiceProvider(Item item);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/PluginChecker.java

@@ -126,6 +126,8 @@ public class PluginChecker {
 
     public static final String BDEE_PLUGIN_ID = "org.talend.designer.bigdata"; //$NON-NLS-1$
 
+    public static final String BD_ROUTINE_PLUGIN_ID = "org.talend.designer.routines.bigdata"; //$NON-NLS-1$
+
     public static final String DOCKER_PLUGIN_ID = "org.talend.designer.docker"; //$NON-NLS-1$
 
     public static final String HELP_DI_EE_PLUGIN_ID = "org.talend.help.dataprep"; //$NON-NLS-1$
@@ -136,6 +138,10 @@ public class PluginChecker {
 
     public static final String HELP_PLUGIN_ID = "org.talend.help"; //$NON-NLS-1$
 
+    public static final String APACHE_CXF_PLUGIN_ID = "org.talend.libraries.apache.cxf"; //$NON-NLS-1$
+
+    public static final String SWTBOT_PLUGIN_ID = "org.eclipse.swtbot.eclipse.core"; //$NON-NLS-1$
+
     /**
      * Check if specific plug-in is loaded.
      *
@@ -364,6 +370,14 @@ public class PluginChecker {
         return isPluginLoaded(ESBSE_PLUGIN_ID);
     }
 
+    public static boolean isBigdataRoutineLoaded() {
+        return isPluginLoaded(BD_ROUTINE_PLUGIN_ID);
+    }
+
+    public static boolean isSWTBotLoaded() {
+        return isPluginLoaded(SWTBOT_PLUGIN_ID);
+    }
+
     public static String getBundlePath(String bundleName) {
         String bundlePath = ""; //$NON-NLS-1$
         Bundle refBundle = Platform.getBundle(bundleName);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/EDatabase4DriverClassName.java

@@ -67,8 +67,8 @@ public enum EDatabase4DriverClassName {
     // if change the order, it might break the audit for PSQL.
     PSQL(EDatabaseTypeName.PSQL, "org.postgresql.Driver"), //$NON-NLS-1$
     PLUSPSQL(EDatabaseTypeName.PLUSPSQL, "org.postgresql.Driver"), //$NON-NLS-1$
-    GREENPLUM(EDatabaseTypeName.GREENPLUM, "org.postgresql.Driver"), //$NON-NLS-1$
-
+    GREENPLUM(EDatabaseTypeName.GREENPLUM, "com.pivotal.jdbc.GreenplumDriver"), //$NON-NLS-1$
+    GREENPLUM_PSQL(EDatabaseTypeName.GREENPLUM, "org.postgresql.Driver"), //$NON-NLS-1$
     SAS(EDatabaseTypeName.SAS, "com.sas.rio.MVADriver"), //$NON-NLS-1$
     SAPHana(EDatabaseTypeName.SAPHana, "com.sap.db.jdbc.Driver"), //$NON-NLS-1$
     SQLITE(EDatabaseTypeName.SQLITE, "org.sqlite.JDBC"), //$NON-NLS-1$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/EImpalaDriver.java

@@ -0,0 +1,84 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.database;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.talend.core.runtime.hd.hive.HiveMetadataHelper;
+
+/**
+ * DOC hzhao  class global comment. Detailled comment
+ */
+public enum EImpalaDriver {
+
+    HIVE2("HIVE2", "HIVE2", "org.apache.hive.jdbc.HiveDriver"),
+    IMPALA40("IMPALA40", "IMPALA40", "com.cloudera.impala.jdbc4.Driver"),
+    IMPALA41("IMPALA41", "IMPALA41", "com.cloudera.impala.jdbc41.Driver");
+
+    EImpalaDriver(String displayName, String name, String driver) {
+        this.displayName = displayName;
+        this.name = name;
+        this.driver = driver;
+    }
+
+    private String displayName;
+
+    private String name;
+
+    private String driver;
+
+    public String getDisplayName() {
+        return displayName;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getDriver() {
+        return driver;
+    }
+
+    public static boolean isSupport(String distribution, String version, boolean byDisplay, String supportMethodName) {
+        return HiveMetadataHelper.doSupportMethod(distribution, version, byDisplay, supportMethodName);
+    }
+
+    public static String[] getImpalaDriverDisplay(String distribution, String version, boolean byDisplay) {
+        List<String> list = new ArrayList<>(0);
+        for (EImpalaDriver driver : EImpalaDriver.values()) {
+            if (isSupport(distribution, version, byDisplay, "doSupportImpalaConnector")) {
+                list.add(driver.getDisplayName());
+            }
+        }
+        return list.toArray(new String[0]);
+    }
+
+    public static EImpalaDriver getByDisplay(String display) {
+        for (EImpalaDriver driver : EImpalaDriver.values()) {
+            if (driver.getDisplayName().equals(display)) {
+                return driver;
+            }
+        }
+        return null;
+    }
+
+    public static EImpalaDriver getByName(String name) {
+        for (EImpalaDriver driver : EImpalaDriver.values()) {
+            if (driver.getName().equals(name)) {
+                return driver;
+            }
+        }
+        return null;
+    }
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/ConnParameterKeys.java

@@ -163,6 +163,18 @@ public class ConnParameterKeys {
 
     public static final String CONN_PARA_KEY_HIVE_AUTHENTICATION_MAPRTICKET_DURATION = "CONN_PARA_KEY_HIVE_AUTHENTICATION_MAPRTICKET_DURATION";//$NON-NLS-1$
 
+    public static final String CONN_PARA_KEY_HIVE_GOOGLE_PROJECT_ID = "CONN_PARA_KEY_HIVE_GOOGLE_PROJECT_ID";//$NON-NLS-1$
+
+    public static final String CONN_PARA_KEY_HIVE_GOOGLE_CLUSTER_ID = "CONN_PARA_KEY_HIVE_GOOGLE_CLUSTER_ID";//$NON-NLS-1$
+
+    public static final String CONN_PARA_KEY_HIVE_GOOGLE_REGION = "CONN_PARA_KEY_HIVE_GOOGLE_REGION";//$NON-NLS-1$
+
+    public static final String CONN_PARA_KEY_HIVE_GOOGLE_JARS_BUCKET = "CONN_PARA_KEY_HIVE_GOOGLE_JARS_BUCKET";//$NON-NLS-1$
+
+    public static final String CONN_PARA_KEY_HIVE_AUTHENTICATION_DEFINE_PATH_TO_GOOGLE_CREDENTIALS = "CONN_PARA_KEY_HIVE_AUTHENTICATION_DEFINE_PATH_TO_GOOGLE_CREDENTIALS";//$NON-NLS-1$
+
+    public static final String CONN_PARA_KEY_HIVE_AUTHENTICATION_PATH_TO_GOOGLE_CREDENTIALS = "CONN_PARA_KEY_HIVE_AUTHENTICATION_PATH_TO_GOOGLE_CREDENTIALS";//$NON-NLS-1$
+
     /**
      * The key is for metastore server.
      */
@@ -267,6 +279,8 @@ public class ConnParameterKeys {
 
     public static final String IMPALA_AUTHENTICATION_PRINCIPLA = "IMPALA_AUTHENTICATION_PRINCIPLA";//$NON-NLS-1$
 
+    public static final String IMPALA_DRIVER = "IMPALA_DRIVER";
+
     /**
      * Google Dataproc keys.
      */

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/DatabaseConnStrUtil.java

@@ -114,6 +114,13 @@ public class DatabaseConnStrUtil {
                 && EDatabaseVersion4Drivers.SYBASEIQ_16_SA.getVersionValue().equals(dbVersion)) {
         	connStr = EDatabaseConnTemplate.SYBASEASE_16_SA;
         }
+        if (EDatabaseTypeName.GREENPLUM.getDisplayName().equals(dbType)) {
+            if (EDatabaseVersion4Drivers.GREENPLUM.getVersionValue().equals(dbVersion)) {
+                connStr = EDatabaseConnTemplate.GREENPLUM;
+            }else if(EDatabaseVersion4Drivers.GREENPLUM_PSQL.getVersionValue().equals(dbVersion)) {
+                connStr = EDatabaseConnTemplate.GREENPLUM_PSQL;
+            }
+        }
         EDatabaseVersion4Drivers version = EDatabaseVersion4Drivers.indexOfByVersion(dbVersion);
         if (connStr != null) {
             String s = connStr.getUrlTemplate(version);
@@ -224,6 +231,7 @@ public class DatabaseConnStrUtil {
         if (template.startsWith(DbConnStrForHive.URL_HIVE_2_TEMPLATE)) {
             url = getImpalaURLString(false, server, port, sidOrDatabase, impalaPrincipal);
         }
+        url = attachAdditionalHiveParameters(url, dbConn, false);
         return url;
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/template/EDatabaseConnTemplate.java

@@ -23,6 +23,7 @@ import org.talend.core.database.EDatabaseTypeName;
 import org.talend.core.database.conn.version.EDatabaseVersion4Drivers;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.runtime.services.IGenericDBService;
+import org.talend.core.runtime.services.IGenericWizardService;
 
 /**
  * cli class global comment. Detailled comment
@@ -53,6 +54,11 @@ public enum EDatabaseConnTemplate {
             "5432")), //$NON-NLS-1$
 
     GREENPLUM(new DbConnStr(EDatabaseTypeName.GREENPLUM, //
+            "jdbc:pivotal:greenplum://<host>:<port>;;DatabaseName=<sid>;<property>", //$NON-NLS-1$
+            "5432")), //$NON-NLS-1$
+    
+
+    GREENPLUM_PSQL(new DbConnStr(EDatabaseTypeName.GREENPLUM, //
             "jdbc:postgresql://<host>:<port>/<sid>?<property>", //$NON-NLS-1$
             "5432")), //$NON-NLS-1$
 
@@ -326,6 +332,16 @@ public enum EDatabaseConnTemplate {
                 databaseType.add(typeName);
             }
         }
+        // add additional jdbc (actually JDBC RepositoryObjectType)
+        if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericWizardService.class)) {
+            IGenericWizardService service = GlobalServiceRegister.getDefault().getService(IGenericWizardService.class);
+            if (service != null) {
+                List<String> allAdditionalJDBCTypes = service.getAllAdditionalJDBCTypes();
+                if (!allAdditionalJDBCTypes.isEmpty()) {
+                    databaseType.addAll(allAdditionalJDBCTypes);
+                }
+            }
+        }
         if (sort) {
             String[] sortedArray = databaseType.toArray(new String[0]);
             Arrays.sort(sortedArray, new Comparator<String>() {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/version/EDatabaseVersion4Drivers.java

@@ -58,7 +58,7 @@ public enum EDatabaseVersion4Drivers {
     HSQLDB_SERVER(new DbVersion4Drivers(EDatabaseTypeName.HSQLDB_SERVER, "hsqldb.jar")), //$NON-NLS-1$
     HSQLDB_WEBSERVER(new DbVersion4Drivers(EDatabaseTypeName.HSQLDB_WEBSERVER, "hsqldb.jar")), //$NON-NLS-1$
 
-    H2(new DbVersion4Drivers(EDatabaseTypeName.H2, "h2-1.3.160.jar")), //$NON-NLS-1$
+    H2(new DbVersion4Drivers(EDatabaseTypeName.H2, "h2-1.4.198.jar")), //$NON-NLS-1$
 
     //
     JAVADB_EMBEDED(new DbVersion4Drivers(EDatabaseTypeName.JAVADB_EMBEDED, "derby.jar")), //$NON-NLS-1$
@@ -99,15 +99,15 @@ public enum EDatabaseVersion4Drivers {
     VERTICA_3(new DbVersion4Drivers(EDatabaseTypeName.VERTICA, "VERTICA 3 (Deprecated)", "VERTICA_3", "vertica_3.0_jdk_5.jar")), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
 
     // for bug 0017930
-
-    GREENPLUM(new DbVersion4Drivers(EDatabaseTypeName.GREENPLUM, "postgresql-8.4-703.jdbc4.jar")), //$NON-NLS-1$
+    GREENPLUM_PSQL(new DbVersion4Drivers(EDatabaseTypeName.GREENPLUM,"PostgreSQL", "POSTGRESQL", "postgresql-8.4-703.jdbc4.jar")), //$NON-NLS-1$
+    GREENPLUM(new DbVersion4Drivers(EDatabaseTypeName.GREENPLUM,"Greenplum", "GREENPLUM", "greenplum-5.1.4.000275.jar")), //$NON-NLS-1$
 //    PSQL_V10(new DbVersion4Drivers(EDatabaseTypeName.PSQL, "v10", "V10", "postgresql-42.2.5.jar")),
-    PSQL_V9_X(new DbVersion4Drivers(EDatabaseTypeName.PSQL, "v9 and later", "V9_X", "postgresql-42.2.9.jar")), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+    PSQL_V9_X(new DbVersion4Drivers(EDatabaseTypeName.PSQL, "v9 and later", "V9_X", "postgresql-42.2.14.jar")), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
     PSQL_PRIOR_TO_V9(new DbVersion4Drivers(EDatabaseTypeName.PSQL, "Prior to v9", "PRIOR_TO_V9", "postgresql-8.4-703.jdbc4.jar")), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
 
     PLUSPSQL_PRIOR_TO_V9(new DbVersion4Drivers(EDatabaseTypeName.PLUSPSQL,
             "Prior to v9", "PRIOR_TO_V9", "postgresql-8.4-703.jdbc4.jar")), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-    PLUSPSQL_V9_X(new DbVersion4Drivers(EDatabaseTypeName.PLUSPSQL, "v9 and later", "V9_X", "postgresql-9.4-1201.jdbc41.jar")), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+    PLUSPSQL_V9_X(new DbVersion4Drivers(EDatabaseTypeName.PLUSPSQL, "v9 and later", "V9_X", "postgresql-42.2.14.jar")), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
     IBMDB2(new DbVersion4Drivers(EDatabaseTypeName.IBMDB2, new String[] { "db2jcc4.jar", "db2jcc_license_cu.jar", //$NON-NLS-1$ //$NON-NLS-2$
             "db2jcc_license_cisuz.jar" })), //$NON-NLS-1$
     IBMDB2ZOS(new DbVersion4Drivers(EDatabaseTypeName.IBMDB2ZOS, new String[] { "db2jcc4.jar", "db2jcc_license_cu.jar", //$NON-NLS-1$ //$NON-NLS-2$
@@ -166,7 +166,7 @@ public enum EDatabaseVersion4Drivers {
     REDSHIFT(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT, "redshift", "REDSHIFT", //$NON-NLS-1$ //$NON-NLS-2$
             "redshift-jdbc42-no-awssdk-1.2.37.1061.jar")), //$NON-NLS-1$
     REDSHIFT_SSO(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT_SSO, "redshift sso", "REDSHIFT_SSO", //$NON-NLS-1$ //$NON-NLS-2$
-            new String[] { "redshift-jdbc42-no-awssdk-1.2.37.1061.jar", "aws-java-sdk-1.11.729.jar", "jackson-core-2.10.1.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+            new String[] { "redshift-jdbc42-no-awssdk-1.2.37.1061.jar", "aws-java-sdk-1.11.848.jar", "jackson-core-2.10.1.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
                     "jackson-databind-2.10.1.jar", "jackson-annotations-2.10.1.jar", "httpcore-4.4.11.jar", "httpclient-4.5.9.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$//$NON-NLS-4$
                     "joda-time-2.8.1.jar", "commons-logging-1.2.jar", "commons-codec-1.11.jar" })), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/HadoopConstants.java

@@ -20,6 +20,8 @@ package org.talend.core.hadoop;
 public class HadoopConstants {
 
     public static final String SPARK_LOCAL_MODE = "SPARK_LOCAL_MODE";
+    
+    public static final String SPARK_LOCAL_VERSION = "SPARK_LOCAL_VERSION";
 
     public static final String SPARK_MODE = "SPARK_MODE";
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/components/ComponentUtilities.java

@@ -81,6 +81,11 @@ public final class ComponentUtilities {
 
     public static void setNodeValue(NodeType node, String name, String value) {
         ElementParameterType property = getNodeProperty(node, name);
+        
+        if (property == null) {
+            throw new IllegalArgumentException( "The component node "+node.getComponentName()+" doesn't have the property "+name );
+        }
+        
         property.setValue(value);
     }
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/components/IComponentsFactory.java

@@ -111,4 +111,6 @@ public interface IComponentsFactory {
     public Map<String, File> getComponentsProvidersFolder();
 
     public List<ComponentProviderInfo> getComponentsProvidersInfo();
+    
+    public String getCustomComponentBundlePath();
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/context/ContextUtils.java

@@ -12,6 +12,7 @@
 // ============================================================================
 package org.talend.core.model.context;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -47,12 +48,14 @@ import org.talend.core.model.properties.ContextItem;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.ProcessItem;
+import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.cwm.helper.ResourceHelper;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextType;
 import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
+import org.talend.migration.IMigrationTask.ExecutionResult;
 import org.talend.repository.model.IProxyRepositoryFactory;
 
 /**
@@ -69,8 +72,9 @@ public class ContextUtils {
             "static", "void", "class", "finally", "long", "strictfp", "volatile", "const", "float", "native", "super", "while")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$ //$NON-NLS-9$ //$NON-NLS-10$ //$NON-NLS-11$ //$NON-NLS-12$
 
     private static final Set<String> SECURE_SENSITIVE_CONTEXT_NAMES_EXP = new HashSet<String>(Arrays.asList("resource_flow_temp_folder", "resource_webhook_payload", "resource_file_[\\w]+", 
-           "resource_directory_[\\w]+", "connection_[a-zA-Z0-9]+_[\\w]"));
+           "resource_directory_[\\w]+", "connection_[a-zA-Z0-9]+_[\\w]+"));
     
+
     /**
      *
      * ggu Comment method "isJavaKeyWords".
@@ -88,7 +92,7 @@ public class ContextUtils {
         }
         return false;
     }
-    
+
     /**
     *
     * ggu Comment method "isSecureSensitiveParam".
@@ -241,7 +245,6 @@ public class ContextUtils {
         return parameterType;
     }
 
-    @SuppressWarnings("unchecked")
     private static boolean checkObject(Object obj) {
         if (obj == null) {
             return true;
@@ -288,7 +291,7 @@ public class ContextUtils {
 
         IProxyRepositoryFactory factory = CoreRuntimePlugin.getInstance().getProxyRepositoryFactory();
         try {
-            final IRepositoryViewObject lastVersion = factory.getLastVersion(contextId);
+            final IRepositoryViewObject lastVersion = factory.getLastVersion(contextId, ERepositoryObjectType.CONTEXT);
             if (lastVersion != null) {
                 final Item item = lastVersion.getProperty().getItem();
                 if (item != null && item instanceof ContextItem) {
@@ -505,9 +508,13 @@ public class ContextUtils {
             return null;
         }
 
-        IProxyRepositoryFactory factory = CoreRuntimePlugin.getInstance().getProxyRepositoryFactory();
+        List<ERepositoryObjectType> possibleTypes = new ArrayList<ERepositoryObjectType>();
+        possibleTypes.add(ERepositoryObjectType.CONTEXT);
+        possibleTypes.addAll(ERepositoryObjectType.getAllTypesOfJoblet());
+        possibleTypes.addAll(ERepositoryObjectType.getAllTypesOfProcess());
         try {
-            final IRepositoryViewObject lastVersion = factory.getLastVersion(contextId);
+            IRepositoryViewObject lastVersion = CoreRuntimePlugin.getInstance().getProxyRepositoryFactory()
+                    .getLastVersion(contextId, possibleTypes);
             if (lastVersion != null) {
                 final Item item = lastVersion.getProperty().getItem();
                 if (item != null) {
@@ -587,7 +594,7 @@ public class ContextUtils {
                 return false;
             }
             // need check the raw value, because in sourceParam, it's raw
-            if (!sourceParam.getValue().equals(targetParamType.getRawValue())) {
+            if (!StringUtils.equals(sourceParam.getValue(), targetParamType.getRawValue())) {
                 return false;
             }
 
@@ -601,7 +608,8 @@ public class ContextUtils {
             return false;
         }
 
-        // preference name must match TalendDesignerPrefConstants.PROPAGATE_CONTEXT_VARIABLE
+        // preference name must match
+        // TalendDesignerPrefConstants.PROPAGATE_CONTEXT_VARIABLE
         return Boolean.parseBoolean(
                 CoreRuntimePlugin.getInstance().getDesignerCoreService().getPreferenceStore("propagateContextVariable")); //$NON-NLS-1$
     }
@@ -1056,4 +1064,70 @@ public class ContextUtils {
         }
         return false;
     }
+
+    public static ExecutionResult doCreateContextLinkMigration(Item item) {
+        IProxyRepositoryFactory proxyRepositoryFactory = CoreRuntimePlugin.getInstance().getProxyRepositoryFactory();
+        boolean modified = false, hasLinkFile = false;
+        try {
+            List<ContextType> contextTypeList = ContextUtils.getAllContextType(item);
+            if (contextTypeList != null && contextTypeList.size() > 0) {
+                for (ContextType contextType : contextTypeList) {
+                    for (Object obj : contextType.getContextParameter()) {
+                        if (obj instanceof ContextParameterType) {
+                            ContextParameterType paramType = (ContextParameterType) obj;
+                            if (ContextUtils.isBuildInParameter(paramType)) {
+                                if (StringUtils.isEmpty(paramType.getInternalId())) {
+                                    paramType.setInternalId(proxyRepositoryFactory.getNextId());
+                                    modified = true;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            hasLinkFile = ContextLinkService.getInstance().saveContextLink(item);
+        } catch (Exception ex) {
+            ExceptionHandler.process(ex);
+            return ExecutionResult.FAILURE;
+        }
+        if (modified || hasLinkFile) {
+            try {
+                if (modified) {
+                    proxyRepositoryFactory.save(item, true);
+                }
+                return ExecutionResult.SUCCESS_NO_ALERT;
+            } catch (Exception ex) {
+                ExceptionHandler.process(ex);
+                return ExecutionResult.FAILURE;
+            }
+        }
+        return ExecutionResult.NOTHING_TO_DO;
+    }
+
+    public static ExecutionResult doCreateContextLinkMigration(ERepositoryObjectType repositoryType, Item item) {
+        if (item != null && getAllSupportContextLinkTypes().contains(repositoryType)) {
+            return doCreateContextLinkMigration(item);
+        }
+        return ExecutionResult.NOTHING_TO_DO;
+    }
+
+    public static List<ERepositoryObjectType> getAllSupportContextLinkTypes() {
+        List<ERepositoryObjectType> toReturn = new ArrayList<ERepositoryObjectType>();
+        toReturn.addAll(ERepositoryObjectType.getAllTypesOfProcess());
+        toReturn.addAll(ERepositoryObjectType.getAllTypesOfProcess2());
+        toReturn.addAll(ERepositoryObjectType.getAllTypesOfTestContainer());
+        toReturn.addAll(getAllMetaDataType());
+        return toReturn;
+    }
+
+    private static List<ERepositoryObjectType> getAllMetaDataType() {
+        List<ERepositoryObjectType> list = new ArrayList<ERepositoryObjectType>();
+        ERepositoryObjectType[] allTypes = (ERepositoryObjectType[]) ERepositoryObjectType.values();
+        for (ERepositoryObjectType object : allTypes) {
+            if (object.isChildTypeOf(ERepositoryObjectType.METADATA)) {
+                list.add(object);
+            }
+        }
+        return list;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/context/JobContextManager.java

@@ -320,7 +320,8 @@ public class JobContextManager implements IContextManager {
                 contextParam = new JobContextParameter();
                 contextParam.setContext(context);
                 contextParam.setName(contextParamType.getName());
-                contextParam.setPrompt(contextParamType.getPrompt());
+                contextParam.setPrompt(
+                        contextParamType.getPrompt() == null ? (contextParamType.getName() + "?") : contextParamType.getPrompt());
                 contextParam.setInternalId(contextParamType.getInternalId());
                 originalParamerters.add(contextParam.getName());
                 boolean exists = true;

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/context/link/AbstractItemContextLinkService.java

@@ -18,15 +18,18 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.lang3.StringUtils;
+import org.eclipse.emf.common.util.EList;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.model.context.ContextUtils;
 import org.talend.core.model.properties.ContextItem;
 import org.talend.core.model.properties.Item;
+import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.cwm.helper.ResourceHelper;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextType;
+import org.talend.repository.ProjectManager;
 import org.talend.repository.model.IRepositoryService;
 
 
@@ -106,16 +109,26 @@ public abstract class AbstractItemContextLinkService implements IItemContextLink
         }
 
         Item contextItem = tempCache.get(repositoryContextId);
+        IRepositoryService repositoryService = GlobalServiceRegister.getDefault().getService(IRepositoryService.class);
         if (contextItem == null) {
             contextItem = ContextUtils.getRepositoryContextItemById(repositoryContextId);
             tempCache.put(repositoryContextId, contextItem);
+            if (contextItem != null && !(contextItem instanceof ContextItem)
+                    && ProjectManager.getInstance().isInCurrentMainProject(contextItem)
+                    && checkRepoItemContextParamInternalId(contextItem)) {
+                try {
+                    repositoryService.getProxyRepositoryFactory().save(contextItem, true); // This should only using for migration phase                                                         // case the internal id is null
+                } catch (Exception ex) {
+                    ExceptionHandler.process(ex);
+                }
+            }
         }
         if (contextItem != null) {
             ContextType contextType = ContextUtils.getContextTypeByName(contextItem, contextName);
             ContextParameterType repoContextParameterType = ContextUtils.getContextParameterTypeByName(contextType, paramName);
             if (repoContextParameterType != null && repoContextParameterType.eResource() == null) {
                 // processItem save before than contextItem, caused eResource null
-                IRepositoryService repositoryService = GlobalServiceRegister.getDefault().getService(IRepositoryService.class);
+
                 try {
                     repositoryService.getProxyRepositoryFactory().save(contextItem, false);
                 } catch (PersistenceException e) {
@@ -170,6 +183,30 @@ public abstract class AbstractItemContextLinkService implements IItemContextLink
         }
         return null;
     }
+    
+    protected boolean checkRepoItemContextParamInternalId(Item item) {
+        boolean isModified = false;
+        EList<?> contextTypeList = ContextUtils.getAllContextType(item);
+        if (contextTypeList != null) {
+            for (Object typeObj : contextTypeList) {
+                if (typeObj instanceof ContextType) {
+                    ContextType type = (ContextType) typeObj;
+                    for (Object obj : type.getContextParameter()) {
+                        if (obj instanceof ContextParameterType) {
+                            ContextParameterType contextParam = (ContextParameterType) obj;
+                            if (ContextUtils.isBuildInParameter(contextParam)
+                                    && StringUtils.isEmpty(contextParam.getInternalId())) {
+                                contextParam
+                                        .setInternalId(CoreRuntimePlugin.getInstance().getProxyRepositoryFactory().getNextId());
+                                isModified = true;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return isModified;
+    }
 }
 
 

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/general/ILibrariesService.java

@@ -20,6 +20,7 @@ import java.util.Set;
 
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.talend.commons.exception.BusinessException;
+import org.talend.core.GlobalServiceRegister;
 import org.talend.core.IService;
 import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
 import org.talend.core.model.process.IElement;
@@ -27,6 +28,7 @@ import org.talend.core.model.process.INode;
 import org.talend.core.model.process.IProcess;
 import org.talend.core.model.process.Problem;
 import org.talend.core.model.repository.ERepositoryObjectType;
+import org.talend.core.model.routines.CodesJarInfo;
 import org.talend.designer.codegen.PigTemplate;
 
 /**
@@ -116,9 +118,15 @@ public interface ILibrariesService extends IService {
 
     Set<ModuleNeeded> getCodesModuleNeededs(ERepositoryObjectType type);
 
+    Set<ModuleNeeded> getAllCodesJarModuleNeededs();
+
+    Set<ModuleNeeded> getCodesJarModuleNeededs(CodesJarInfo info);
+
     List<ModuleNeeded> getModuleNeeded(String id, boolean isGroup);
 
     public void deployProjectLibrary(File source) throws IOException;
+    
+    public void setForceReloadCustomUri();
 
     /**
      * Listener used to fire that libraries status has been changed (new lib or new check install).
@@ -131,4 +139,14 @@ public interface ILibrariesService extends IService {
         public void afterChangingLibraries();
     }
 
+    public static ILibrariesService get() {
+        GlobalServiceRegister gsr = GlobalServiceRegister.getDefault();
+        if (gsr != null) {
+            if (gsr.isServiceRegistered(ILibrariesService.class)) {
+                return gsr.getService(ILibrariesService.class);
+            }
+        }
+        return null;
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/general/ModuleNeeded.java

@@ -28,6 +28,7 @@ import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenConstants;
 import org.talend.core.runtime.maven.MavenUrlHelper;
+import org.talend.core.utils.TalendQuoteUtils;
 
 /**
  * This bean is use to manage needed moduless (perl) and libraries (java).<br/>
@@ -110,6 +111,15 @@ public class ModuleNeeded {
 
     }
 
+    public static ModuleNeeded newInstance(String context, String value, String informationMsg, boolean required) {
+        String val = TalendQuoteUtils.removeQuotesIfExist(value);
+        if (val.startsWith(MavenUrlHelper.MVN_PROTOCOL)) {
+            return new ModuleNeeded(context, informationMsg, required, val);
+        }
+        // won't do migration for old MODULE_LIST but still make it compatible
+        return new ModuleNeeded(context, val, informationMsg, required);
+    }
+
     /**
      * DOC smallet ModuleNeeded constructor comment.
      *

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/IMetadataColumn.java

@@ -125,6 +125,10 @@ public interface IMetadataColumn {
 
     public void setCustom(boolean custom);
 
+    public boolean isIgnoreCustomSort();
+
+    public void setIgnoreCustomSort(boolean ignoreCustomSort);
+
     public void setCustomId(int customId); // only for custom columns to sort them
 
     public int getCustomId(); // only for custom columns to sort them

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataColumn.java

@@ -53,6 +53,8 @@ public class MetadataColumn implements IMetadataColumn, Cloneable {
 
     private boolean custom = false;
 
+    private boolean ignoreCustomSort = false;
+
     private boolean readOnly = false;
 
     private int customId = 0;
@@ -574,6 +576,24 @@ public class MetadataColumn implements IMetadataColumn, Cloneable {
         this.custom = custom;
     }
 
+    /**
+     * Getter for ignoreCustomSort.
+     * 
+     * @return the ignoreCustomSort
+     */
+    public boolean isIgnoreCustomSort() {
+        return ignoreCustomSort;
+    }
+
+    /**
+     * Sets the ignoreCustomSort.
+     * 
+     * @param ignoreCustomSort the ignoreCustomSort to set
+     */
+    public void setIgnoreCustomSort(boolean ignoreCustomSort) {
+        this.ignoreCustomSort = ignoreCustomSort;
+    }
+
     /**
      * Getter for readOnly.
      *

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataTable.java

@@ -360,7 +360,7 @@ public class MetadataTable implements IMetadataTable, Cloneable {
 
         for (int i = 0; i < listColumns.size(); i++) {
             IMetadataColumn column = listColumns.get(i);
-            if (column.isCustom()) {
+            if (column.isCustom() && !column.isIgnoreCustomSort()) {
                 customColumns.add(column);
             }
         }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataTalendType.java

@@ -13,6 +13,7 @@
 package org.talend.core.model.metadata;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.net.URL;
 import java.util.ArrayList;
@@ -31,8 +32,13 @@ import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.StringUtils;
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.resources.ProjectScope;
+import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.Path;
@@ -488,6 +494,79 @@ public final class MetadataTalendType {
         }
     }
 
+    private static String getSha1OfFile(File file) {
+        String sha1 = null;
+        FileInputStream fileInputStream = null;
+        try {
+            fileInputStream = new FileInputStream(file);
+            sha1 = DigestUtils.shaHex(fileInputStream);
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        } finally {
+            if (fileInputStream != null) {
+                try {
+                    fileInputStream.close();
+                } catch (IOException e) {
+                    e.printStackTrace(); // Just print the exception for debug.
+                }
+            }
+        }
+        return sha1;
+    }
+    
+    public static void copyFile(File in, IFile out) throws CoreException, IOException {
+        FileInputStream fis = new FileInputStream(in);
+        if (out.exists()) {
+            out.setContents(fis, true, false, null);
+        } else {
+            out.create(fis, true, null);
+        }
+        fis.close();
+    }
+
+    public static void syncNewMappingFileToProject() throws SystemException {
+        try {
+            File sysMappingFiles = new File(MetadataTalendType.getSystemForderURLOfMappingsFile().getPath());
+            IFolder projectMappingFolder = ResourceUtils.getProject(ProjectManager.getInstance().getCurrentProject())
+                    .getFolder(MetadataTalendType.PROJECT_MAPPING_FOLDER);
+            File projectMappingFiles = new File(projectMappingFolder.getLocationURI());
+            if (!sysMappingFiles.exists() || !projectMappingFiles.exists()) {
+                return;
+            }
+            for (File sysMapping : sysMappingFiles.listFiles()) {
+                IFile projectMapping = projectMappingFolder.getFile(sysMapping.getName());
+                if (projectMapping.exists() && StringUtils.equals(sysMapping.getName(), "mapping_Greenplum.xml")) {
+
+                    String sha1OfFile = DigestUtils.shaHex(projectMapping.getContents());
+                    String shalOfOldSystem = "8431f19215dacb3caa126778ae695954552cce2a";
+                    if (StringUtils.equals(sha1OfFile, shalOfOldSystem)) {
+                        copyFile(sysMapping, projectMapping);
+                    }
+                }
+                if (!projectMapping.exists()) {
+                    FileInputStream fis = null;
+                    try {
+                        fis = new FileInputStream(sysMapping);
+                        projectMapping.create(fis, true, null);
+                    } catch (CoreException coreExc) {
+                        throw new SystemException(coreExc);
+                    } finally {
+                        if (fis != null) {
+                            try {
+                                fis.close();
+                            } catch (IOException ioExc) {
+                                throw new SystemException(ioExc);
+                            }
+                        }
+                    }
+                }
+            }
+
+        } catch (Exception e) {
+            throw new SystemException(e);
+        }
+    }
+
     /**
      *
      * Load db types and mapping with the current activated language (Java, Perl, ...).

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolAvroHelper.java

@@ -25,8 +25,10 @@ import org.apache.avro.SchemaBuilder.FieldAssembler;
 import org.apache.avro.SchemaBuilder.FieldBuilder;
 import org.apache.avro.SchemaBuilder.PropBuilder;
 import org.apache.avro.SchemaBuilder.RecordBuilder;
+import org.apache.avro.SchemaParseException;
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
 import org.eclipse.emf.common.util.EList;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.GlobalServiceRegister;
@@ -49,6 +51,11 @@ import orgomg.cwm.objectmodel.core.TaggedValue;
  */
 public final class MetadataToolAvroHelper {
 
+    private static Logger LOGGER = Logger.getLogger(MetadataToolAvroHelper.class);
+
+    private static final String COLUMN = "Column";
+
+    private static final String TALEND_DB_COLUMN_NAME = "talend.field.dbColumnName";
     /**
      * @return An Avro schema with enriched properties from the incoming metadata table.
      */
@@ -65,7 +72,7 @@ public final class MetadataToolAvroHelper {
                 dynamicPosition = i;
                 dynColumn = column;
             } else {
-                fa = convertToAvro(fa, column);
+                fa = convertToAvro(fa, column, i);
             }
             i++;
         }
@@ -136,7 +143,7 @@ public final class MetadataToolAvroHelper {
      * Build a field into a schema using enriched properties from the incoming column.
      */
     private static FieldAssembler<Schema> convertToAvro(FieldAssembler<Schema> fa,
-            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
+            org.talend.core.model.metadata.builder.connection.MetadataColumn in, int i) {
         ICoreService coreService = (ICoreService) GlobalServiceRegister.getDefault().getService(ICoreService.class);
         String label = in.getLabel();
         if (label != null && coreService != null) {
@@ -236,7 +243,27 @@ public final class MetadataToolAvroHelper {
         }
 
         type = in.isNullable() ? AvroUtils.wrapAsNullable(type) : type;
-        return defaultValue == null ? fb.type(type).noDefault() : fb.type(type).withDefault(defaultValue);
+        FieldAssembler<Schema> returnResult = null;
+        try {
+            if (defaultValue == null) {
+                returnResult = fb.type(type).noDefault();
+            } else {
+                returnResult = fb.type(type).withDefault(defaultValue);
+            }
+        } catch (SchemaParseException e) {
+            // if validation not pass from avro ,then generate a new name like Column0 to pass the validation, but
+            // actually the display name of schema and the column name is using TALEND_DB_COLUMN_NAME
+            String genColumn = COLUMN + i;
+            FieldBuilder<Schema> fbNew = fa.name(genColumn);
+            copyColumnProperties(fbNew, in);
+            if (defaultValue == null) {
+                returnResult = fbNew.type(type).noDefault();
+            } else {
+                returnResult = fbNew.type(type).withDefault(defaultValue);
+            }
+            LOGGER.info(e.getMessage() + ", use " + genColumn + " instead");
+        }
+        return returnResult;
     }
 
     private static Schema getLogicalTypeSchema(org.talend.core.model.metadata.builder.connection.MetadataColumn column) {
@@ -451,19 +478,16 @@ public final class MetadataToolAvroHelper {
             table.setTableType(prop);
         }
 
-        // Add the columns.
-        List<org.talend.core.model.metadata.builder.connection.MetadataColumn> columns = new ArrayList<>(in.getFields().size());
         for (Schema.Field f : in.getFields()) {
-            columns.add(convertFromAvro(f, table));
+            table.getColumns().add(convertFromAvro(f, table));
         }
         boolean isDynamic = AvroUtils.isIncludeAllFields(in);
         if (isDynamic) {
             org.talend.core.model.metadata.builder.connection.MetadataColumn col = convertFromAvroForDynamic(in);
             // get dynamic position
             int dynPosition = Integer.valueOf(in.getProp(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION));
-            columns.add(dynPosition, col);
+            table.getColumns().add(dynPosition, col);
         }
-        table.getColumns().addAll(columns);
         return table;
     }
 
@@ -574,7 +598,12 @@ public final class MetadataToolAvroHelper {
 
         // Set the defaults values to the name (the only information guaranteed to be available in every field).
         col.setId(field.name());
-        col.setLabel(field.name());
+        String dbColumnlable = null;
+        if (MetadataToolHelper.isAllowSpecificCharacters() && null != (dbColumnlable = field.getProp(TALEND_DB_COLUMN_NAME))) {
+            col.setLabel(dbColumnlable);
+        } else {
+            col.setLabel(field.name());
+        }
         col.setName(field.name());
         Schema nonnullable = AvroUtils.unwrapIfNullable(in);
         LogicalType logicalType = LogicalTypes.fromSchemaIgnoreInvalid(nonnullable);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolHelper.java

@@ -280,7 +280,7 @@ public final class MetadataToolHelper {
         return isAllowSpecificCharacters() || Pattern.matches(RepositoryConstants.COLUMN_NAME_PATTERN, name);
     }
 
-    private static boolean isAllowSpecificCharacters() {
+    public static boolean isAllowSpecificCharacters() {
         IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
         return coreUIPluginNode
                 .getBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
@@ -614,6 +614,7 @@ public final class MetadataToolHelper {
                     target.getListColumns().remove(targetColumn);
                     newTargetColumn.setCustom(targetColumn.isCustom());
                     newTargetColumn.setCustomId(targetColumn.getCustomId());
+                    newTargetColumn.setIgnoreCustomSort(targetColumn.isIgnoreCustomSort());
                     columnsTAdd.add(newTargetColumn);
                 }
             }
@@ -773,6 +774,7 @@ public final class MetadataToolHelper {
                     target.getListColumns().remove(targetColumn);
                     newTargetColumn.setCustom(targetColumn.isCustom());
                     newTargetColumn.setCustomId(targetColumn.getCustomId());
+                    newTargetColumn.setIgnoreCustomSort(targetColumn.isIgnoreCustomSort());
                     columnsTAdd.add(newTargetColumn);
                 }
             }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/designerproperties/ComponentToRepositoryProperty.java

@@ -18,6 +18,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
 import org.eclipse.emf.common.util.EList;
 import org.eclipse.jface.dialogs.MessageDialog;
 import org.eclipse.swt.widgets.Display;
@@ -26,9 +27,11 @@ import org.talend.components.api.properties.ComponentProperties;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.database.EDatabase4DriverClassName;
 import org.talend.core.database.EDatabaseTypeName;
+import org.talend.core.database.conn.ConnParameterKeys;
 import org.talend.core.database.conn.DatabaseConnStrUtil;
 import org.talend.core.database.conn.version.EDatabaseVersion4Drivers;
 import org.talend.core.model.components.EComponentType;
+import org.talend.core.model.metadata.Dbms;
 import org.talend.core.model.metadata.IMetadataTable;
 import org.talend.core.model.metadata.MetadataTalendType;
 import org.talend.core.model.metadata.builder.ConvertionHelper;
@@ -70,6 +73,7 @@ import org.talend.core.model.utils.ContextParameterUtils;
 import org.talend.core.model.utils.IDragAndDropServiceHandler;
 import org.talend.core.runtime.i18n.Messages;
 import org.talend.core.runtime.services.IGenericDBService;
+import org.talend.core.runtime.services.IGenericWizardService;
 import org.talend.core.utils.TalendQuoteUtils;
 import org.talend.cwm.helper.ConnectionHelper;
 import org.talend.cwm.helper.PackageHelper;
@@ -152,6 +156,17 @@ public class ComponentToRepositoryProperty {
                     conn.setDbmsId(mapping);
                 }
             }
+            // set default mapping for additional jdbc
+            if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericWizardService.class)) {
+                IGenericWizardService service = GlobalServiceRegister.getDefault().getService(IGenericWizardService.class);
+                if (service != null) {
+                    Dbms dbms4AdditionalJDBC = service.getDbms4AdditionalJDBC(conn.getProductId());
+                    if (dbms4AdditionalJDBC != null) {
+                        conn.setDbmsId(dbms4AdditionalJDBC.getId());
+                    }
+                }
+            }
+
         }
         for (IElementParameter param : node.getElementParameters()) {
             String repositoryValue = param.getRepositoryValue();
@@ -365,6 +380,20 @@ public class ComponentToRepositoryProperty {
                 if (para.getRepositoryValue().endsWith(EDatabaseTypeName.GENERAL_JDBC.getProduct())) {
                     connection.setDatabaseType(EDatabaseTypeName.GENERAL_JDBC.getProduct());
                     connection.setProductId(EDatabaseTypeName.GENERAL_JDBC.getProduct());
+
+                    // additional JDBC e.g. Delta Lake
+                    if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericWizardService.class)) {
+                        IGenericWizardService service = GlobalServiceRegister.getDefault()
+                                .getService(IGenericWizardService.class);
+                        if (service != null) {
+                            String database = service.getDatabseNameByNode(node);
+                            if (StringUtils.isNotBlank(database) && service.getIfAdditionalJDBCDBType(database)) {
+                                connection.setProductId(database);
+                            }
+                        }
+
+                    }
+
                 }
                 // vertica output component have no TYPE ElementParameter .
                 if (para.getRepositoryValue().endsWith(EDatabaseTypeName.VERTICA.getProduct())) {
@@ -667,6 +696,18 @@ public class ComponentToRepositoryProperty {
                 connection.setServerName(value);
             }
         }
+        if ("IMPALA_DRIVER".equals(param.getRepositoryValue())) {
+            String value = getParameterValue(connection, node, param);
+            if (value != null) {
+                connection.getParameters().put(ConnParameterKeys.IMPALA_DRIVER, value);
+            }
+        }
+        if ("IMPALA_ADDITIONAL_JDBC".equals(param.getRepositoryValue())) {
+            String value = getParameterValue(connection, node, param);
+            if (value != null) {
+                connection.getParameters().put(ConnParameterKeys.CONN_PARA_KEY_HIVE_ADDITIONAL_JDBC_SETTINGS, value);
+            }
+        }
 
         if (connection.getDatabaseType().equals(EDatabaseTypeName.ORACLEFORSID.getDisplayName())) {
             setDatabaseValueForOracleSid(connection, node, param);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/designerproperties/RepositoryToComponentProperty.java

@@ -1071,7 +1071,10 @@ public class RepositoryToComponentProperty {
                 return dbVersionString;
             } else if (EDatabaseTypeName.SYBASEASE.getDisplayName().equals(databaseType)) {
                 return dbVersionString;
-            } else {
+            } else if (EDatabaseTypeName.GREENPLUM.getDisplayName().equals(databaseType)) {
+                return dbVersionString;
+                
+            }else {
                 String driverValue = EDatabaseVersion4Drivers.getDriversStr(databaseType, dbVersionString);
                 if (EDatabaseConnTemplate.ORACLE_OCI.getDBDisplayName().equals(databaseType)
                         || EDatabaseConnTemplate.ORACLE_CUSTOM.getDBDisplayName().equals(databaseType)
@@ -1490,21 +1493,24 @@ public class RepositoryToComponentProperty {
         }
 
         if (value.equals("USE_SSL") && (EDatabaseTypeName.HIVE.getDisplayName().equals(databaseType)
-                || EDatabaseTypeName.ORACLE_CUSTOM.getDisplayName().equals(databaseType))) {
+                || EDatabaseTypeName.ORACLE_CUSTOM.getDisplayName().equals(databaseType)
+                || EDatabaseTypeName.IMPALA.getDisplayName().equals(databaseType))) {
             String message = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_USE_SSL);
             return Boolean.parseBoolean(message);
         }
 
         if ((value.equals("SSL_TRUST_STORE") || value.equals("SSL_TRUSTSERVER_TRUSTSTORE"))
                 && (EDatabaseTypeName.HIVE.getDisplayName().equals(databaseType)
-                || EDatabaseTypeName.ORACLE_CUSTOM.getDisplayName().equals(databaseType))) {
+                        || EDatabaseTypeName.ORACLE_CUSTOM.getDisplayName().equals(databaseType)
+                        || EDatabaseTypeName.IMPALA.getDisplayName().equals(databaseType))) {
             return getAppropriateValue(connection,
                     connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_SSL_TRUST_STORE_PATH));
         }
 
         if ((value.equals("SSL_TRUST_STORE_PASSWORD") || value.equals("SSL_TRUSTSERVER_PASSWORD"))
                 && (EDatabaseTypeName.HIVE.getDisplayName().equals(databaseType)
-                || EDatabaseTypeName.ORACLE_CUSTOM.getDisplayName().equals(databaseType))) {
+                        || EDatabaseTypeName.ORACLE_CUSTOM.getDisplayName().equals(databaseType)
+                        || EDatabaseTypeName.IMPALA.getDisplayName().equals(databaseType))) {
             return getAppropriateValue(connection, connection
                     .getValue(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_SSL_TRUST_STORE_PASSWORD), false));
         }
@@ -1755,8 +1761,50 @@ public class RepositoryToComponentProperty {
         if (StringUtils.equals("MAPPING", value)) {//$NON-NLS-1$
             return connection.getDbmsId();
         }
-        return null;
+        if ("IMPALA_ADDITIONAL_JDBC".equals(value)) { //$NON-NLS-1$
+            String additionJdbc = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_HIVE_ADDITIONAL_JDBC_SETTINGS);
+            if (isContextMode(connection, additionJdbc)) {
+                return additionJdbc;
+            } else {
+                return TalendQuoteUtils.addQuotes(additionJdbc);
+            }
+        }
+        if ("IMPALA_DRIVER".equals(value)) {
+            return connection.getParameters().get(ConnParameterKeys.IMPALA_DRIVER);
+        }
 
+        if (value.equals("GOOGLE_PROJECT_ID")) {
+            String projectId = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_HIVE_GOOGLE_PROJECT_ID);
+            return getAppropriateValue(connection, projectId);
+        }
+
+        if (value.equals("GOOGLE_CLUSTER_ID")) {
+            String clusterId = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_HIVE_GOOGLE_CLUSTER_ID);
+            return getAppropriateValue(connection, clusterId);
+        }
+
+        if (value.equals("GOOGLE_REGION")) {
+            String region = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_HIVE_GOOGLE_REGION);
+            return getAppropriateValue(connection, region);
+        }
+
+        if (value.equals("GOOGLE_JARS_BUCKET")) {
+            String jarsBucket = connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_HIVE_GOOGLE_JARS_BUCKET);
+            return getAppropriateValue(connection, jarsBucket);
+        }
+
+        if (value.equals("DEFINE_PATH_TO_GOOGLE_CREDENTIALS")) {
+            String usePathToCredentials = connection.getParameters()
+                    .get(ConnParameterKeys.CONN_PARA_KEY_HIVE_AUTHENTICATION_DEFINE_PATH_TO_GOOGLE_CREDENTIALS);
+            return Boolean.parseBoolean(usePathToCredentials);
+        }
+
+        if (value.equals("PATH_TO_GOOGLE_CREDENTIALS")) {
+            String pathToCredentials = connection.getParameters()
+                    .get(ConnParameterKeys.CONN_PARA_KEY_HIVE_AUTHENTICATION_PATH_TO_GOOGLE_CREDENTIALS);
+            return getAppropriateValue(connection, pathToCredentials);
+        }
+        return null;
     }
 
     private static String getAppropriateValue(Connection connection, String rawValue) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/migration/AbstractItemMigrationTask.java

@@ -146,7 +146,7 @@ public abstract class AbstractItemMigrationTask extends AbstractMigrationTask im
      * this returns all type handled by this migration task even the extended type that could benefit from this
      * migrations
      * */
-    List<ERepositoryObjectType> getAllTypes() {
+    protected List<ERepositoryObjectType> getAllTypes() {
         List<ERepositoryObjectType> declaredTypes = getTypes();
         ArrayList<ERepositoryObjectType> allTypes = new ArrayList<ERepositoryObjectType>(declaredTypes.size());
         allTypes.addAll(declaredTypes);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/EComponentCategory.java

@@ -44,6 +44,7 @@ public enum EComponentCategory {
     DYNAMICS_SETTINGS(Messages.getString("EComponentCategory_dynamicSetting"), 13), //$NON-NLS-1$
     SQL_PATTERN(Messages.getString("EComponentCategory_sqlTemplate"), 14), //$NON-NLS-1$
     BREAKPOINT(Messages.getString("EComponentCategory.breakpoint"), 15), //$NON-NLS-1$
+    BREAKPOINT_CAMEL(Messages.getString("EComponentCategory.breakpoint"), 16), //$NON-NLS-1$
     BASICRUN(Messages.getString("EComponentCategory.basicRun"), 1), //$NON-NLS-1$
     DEBUGRUN(Messages.getString("EComponentCategory.debugRun"), 2), //$NON-NLS-1$
     ADVANCESETTING(Messages.getString("EComponentCategory.advancedSettings"), 3), //$NON-NLS-1$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/EParameterFieldType.java

@@ -113,8 +113,6 @@ public enum EParameterFieldType {
 
     GROUPING_CAMPAIGN_CHOOSER, // htyin added for DQ Matching components
     MULTI_PATTERN, // yyin added TDQ-13437
-    SYNC_NEXUS_BUTTON,
-    CHECK_NEXUS_BUTTON,
     PATTERN_PROPERTY, // yyin, added TDQ-13437
 
     UNIFIED_COMPONENTS,
@@ -126,6 +124,7 @@ public enum EParameterFieldType {
     TACOKIT_BUTTON,
     TACOKIT_SUGGESTABLE_TABLE,
     TACOKIT_VALUE_SELECTION,
+    TACOKIT_TABLE,
     TACOKIT_TEXT_AREA_SELECTION;
 
     public String getName() {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/IElementParameter.java

@@ -232,4 +232,6 @@ public interface IElementParameter {
     public boolean isSerialized();
 
     public void setSerialized(boolean isSerialized);
+
+    public boolean isSelectedFromItemValue();
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/IProcess.java

@@ -131,4 +131,6 @@ public interface IProcess extends IElement {
     public void checkStartNodes();
 
     public String getComponentsType();
+
+    public INode getNodeByUniqueName(String uniqueName);
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/ProcessUtils.java

@@ -21,6 +21,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.lang.StringUtils;
+import org.eclipse.emf.common.util.EList;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.utils.VersionUtils;
@@ -53,6 +54,7 @@ import org.talend.core.utils.BitwiseOptionUtils;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextType;
+import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.MetadataType;
 import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
 import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
@@ -60,6 +62,7 @@ import org.talend.designer.core.model.utils.emf.talendfile.RoutinesParameterType
 import org.talend.designer.runprocess.ItemCacheManager;
 import org.talend.repository.ProjectManager;
 import org.talend.repository.model.IProxyRepositoryFactory;
+import org.apache.commons.text.StringEscapeUtils;
 
 /**
  * DOC bqian class global comment. Detailled comment
@@ -1003,4 +1006,49 @@ public final class ProcessUtils {
         }
         return false;
     }
+
+    public static boolean isChildRouteProcess(IProcess process) {
+        if (process != null)   {
+            for (INode node : process.getGraphicalNodes()) {
+                if ((node.getComponent().getName() != null) && 
+                        (node.getComponent().getName().compareTo("tRouteInput") == 0)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+	public static String escapeJava(String input) {
+        return StringEscapeUtils.escapeJava(input);
+    }
+
+    public static boolean hasJettyEndpoint(ProcessType process) {
+
+		EList<NodeType> nodesList = process.getNode();
+
+		boolean hasJettyEndpoint = hasJettyEndpoint(nodesList);
+
+		return hasJettyEndpoint;
+	}
+
+	private static boolean hasJettyEndpoint(EList<NodeType> nodesList) {
+		for (NodeType node : nodesList) {
+			if ("cMessagingEndpoint".equals(node.getComponentName())) {
+				for (Object elementParameter : node.getElementParameter()) {
+					ElementParameterType elementParameterType = (ElementParameterType)elementParameter;
+
+					String name = elementParameterType.getName();
+					String value = elementParameterType.getValue();
+
+					if ("URI".equals(name) && (value != null && StringUtils.startsWith(value.trim(), "\"jetty:"))) {
+						return true;
+					}
+				}
+			}
+		}
+
+		return false;
+	}
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/relationship/RelationshipItemBuilder.java

@@ -101,6 +101,10 @@ public class RelationshipItemBuilder {
 
     public static final String ROUTINE_RELATION = "routine"; //$NON-NLS-1$
 
+    public static final String ROUTINES_JAR_RELATION = "routinesJar"; //$NON-NLS-1$
+
+    public static final String BEANS_JAR_RELATION = "beansJar"; //$NON-NLS-1$
+
     public static final String MAPPER_RELATION = "mapper"; //$NON-NLS-1$
 
     public static final String SURVIVOR_RELATION = "survivorshipRuleRelation"; //$NON-NLS-1$
@@ -493,7 +497,7 @@ public class RelationshipItemBuilder {
     }
 
     private Set<Relation> getItemsRelatedTo(Map<Relation, Set<Relation>> itemsRelations, String itemId, String version,
-            String relationType) {
+                String relationType) {
 
         Relation itemToTest = new Relation();
         itemId = ProcessUtils.getPureItemId(itemId);
@@ -929,7 +933,10 @@ public class RelationshipItemBuilder {
         }
     }
 
-    private void addRelationShip(Item baseItem, String relatedId, String relatedVersion, String type) {
+    public void addRelationShip(Item baseItem, String relatedId, String relatedVersion, String type) {
+        if (!loaded) {
+            loadRelations();
+        }
         Relation relation = new Relation();
         relation.setId(baseItem.getProperty().getId());
         relation.setType(getTypeFromItem(baseItem));
@@ -945,8 +952,32 @@ public class RelationshipItemBuilder {
             itemRelations.put(relation, new HashSet<Relation>());
         }
         itemRelations.get(relation).add(addedRelation);
+        
+        autoSaveRelations();
     }
 
+	public Set<Relation> getBeanRelations(Collection<Item> items) {
+		Set<Relation> relationships = new HashSet<Relation>();
+		for (Item item : items) {
+	        Relation relation = new Relation();
+	        relation.setId(item.getProperty().getId());
+	        relation.setType(getTypeFromItem(item));
+	        relation.setVersion(item.getProperty().getVersion());
+	        
+	        Map<Relation, Set<Relation>> itemRelations = getRelatedRelations(item);
+	        
+	        Set<Relation> repositoryNode = (Set<Relation>) itemRelations.getOrDefault(relation,  new HashSet<Relation>());
+	        
+	        for (Relation rel : repositoryNode) {
+	        	if (rel.getType().equals("Beans")) {
+	        		relationships.add(rel);
+	        	}
+	        }
+		}
+		
+		return relationships;
+	}
+
     private Map<Relation, Set<Relation>> getRelatedRelations(Item baseItem) {
         Map<Relation, Set<Relation>> itemRelations = currentProjectItemsRelations;
         if (!ProjectManager.getInstance().isInMainProject(getAimProject(), baseItem)) {
@@ -1022,7 +1053,7 @@ public class RelationshipItemBuilder {
             ExceptionHandler.process(e);
         }
     }
-
+    
     private List<ERepositoryObjectType> allSupportedTypes() {
         List<ERepositoryObjectType> toReturn = new ArrayList<ERepositoryObjectType>();
         toReturn.addAll(ERepositoryObjectType.getAllTypesOfProcess());
@@ -1494,4 +1525,5 @@ public class RelationshipItemBuilder {
             return 0;
         }
     }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/repository/ERepositoryObjectType.java

@@ -61,6 +61,7 @@ import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.properties.RegExFileConnectionItem;
 import org.talend.core.model.properties.RoutineItem;
+import org.talend.core.model.properties.RoutinesJarItem;
 import org.talend.core.model.properties.RulesItem;
 import org.talend.core.model.properties.SAPConnectionItem;
 import org.talend.core.model.properties.SQLPatternItem;
@@ -443,8 +444,19 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
 
     public final static ERepositoryObjectType ROUTINES = ERepositoryObjectType.valueOf("ROUTINES"); //$NON-NLS-1$
 
+    public final static ERepositoryObjectType ROUTINESJAR = ERepositoryObjectType.valueOf("ROUTINESJAR"); //$NON-NLS-1$
+
+    /**
+     * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
+     */
     public final static ERepositoryObjectType BEANS = ERepositoryObjectType.valueOf("BEANS"); //$NON-NLS-1$
 
+    /**
+     * <font color="red">This value may be <b>null</b> in some licenses, <b>should add NPE check</b></font>
+     */
+    // TODO need and register it in extension or not?
+    public final static ERepositoryObjectType BEANSJAR = ERepositoryObjectType.valueOf("BEANSJAR"); //$NON-NLS-1$
+
     public final static ERepositoryObjectType METADATA_HEADER_FOOTER = ERepositoryObjectType.valueOf("METADATA_HEADER_FOOTER"); //$NON-NLS-1$
 
     public final static ERepositoryObjectType JOB_SCRIPT = ERepositoryObjectType.valueOf("JOB_SCRIPT"); //$NON-NLS-1$
@@ -936,7 +948,7 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
      * @return
      */
     public static ERepositoryObjectType getItemType(Item item) {
-
+        // FIXME update all inner codes related callers
         ERepositoryObjectType repObjType = getTDQRepObjType(item);
         if (repObjType != null) {
             return repObjType;
@@ -1003,6 +1015,11 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
             // return BEANS;
             // }
 
+            @Override
+            public Object caseRoutinesJarItem(RoutinesJarItem object) {
+                return ROUTINESJAR;
+            }
+
             @Override
             public Object caseJobScriptItem(JobScriptItem object) {
                 return JOB_SCRIPT;
@@ -1414,12 +1431,8 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
         return null;
     }
 
-    public static List<ERepositoryObjectType> getAllTypesOfProcess() {
+    public static List<ERepositoryObjectType> getAllTypesOfBigDataProcess() {
         List<ERepositoryObjectType> allTypes = new ArrayList<ERepositoryObjectType>();
-
-        if (ERepositoryObjectType.PROCESS != null) {
-            allTypes.add(ERepositoryObjectType.PROCESS);
-        }
         if (ERepositoryObjectType.PROCESS_MR != null) {
             allTypes.add(ERepositoryObjectType.PROCESS_MR);
         }
@@ -1432,6 +1445,17 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
         if (ERepositoryObjectType.PROCESS_SPARKSTREAMING != null) {
             allTypes.add(ERepositoryObjectType.PROCESS_SPARKSTREAMING);
         }
+        return allTypes;
+    }
+
+    public static List<ERepositoryObjectType> getAllTypesOfProcess() {
+        List<ERepositoryObjectType> allTypes = new ArrayList<ERepositoryObjectType>();
+
+        allTypes.addAll(getAllTypesOfBigDataProcess());
+
+        if (ERepositoryObjectType.PROCESS != null) {
+            allTypes.add(ERepositoryObjectType.PROCESS);
+        }
         if (ERepositoryObjectType.PROCESS_ROUTE != null) {
             allTypes.add(ERepositoryObjectType.PROCESS_ROUTE);
         }
@@ -1498,11 +1522,66 @@ public class ERepositoryObjectType extends DynaEnum<ERepositoryObjectType> {
         if (ERepositoryObjectType.ROUTINES != null) {
             allTypes.add(ERepositoryObjectType.ROUTINES);
         }
-        ERepositoryObjectType beansType = ERepositoryObjectType.valueOf("BEANS"); //$NON-NLS-1$
-        if (beansType != null) {
-            allTypes.add(beansType);
+        if (ERepositoryObjectType.BEANS != null) {
+            allTypes.add(ERepositoryObjectType.BEANS);
         }
         return allTypes;
     }
 
+    public static List<ERepositoryObjectType> getAllTypesOfCodesJar() {
+        List<ERepositoryObjectType> allTypes = new ArrayList<ERepositoryObjectType>();
+        if (ERepositoryObjectType.ROUTINESJAR != null) {
+            allTypes.add(ERepositoryObjectType.ROUTINESJAR);
+        }
+        if (ERepositoryObjectType.BEANSJAR != null) {
+            allTypes.add(ERepositoryObjectType.BEANSJAR);
+        }
+        return allTypes;
+    }
+
+    public enum CodeTypeEnum {
+        Routines("ROUTINESJAR", "ROUTINES"),
+        Beans("BEANSJAR", "BEANS");
+
+        private String codeJarType;
+
+        private String codeType;
+
+        private CodeTypeEnum(String codeJarType, String codeType) {
+            this.codeJarType = codeJarType;
+            this.codeType = codeType;
+        }
+
+        public static boolean isCodeRepositoryObjectTypeMatch(ERepositoryObjectType jarType, ERepositoryObjectType codeType) {
+            if (jarType == null || codeType == null) {
+                return false;
+            }
+            for (CodeTypeEnum cType : CodeTypeEnum.values()) {
+                if (cType.getCodeJarType().equals(jarType.getType())) {
+                    return cType.getCodeType().equals(codeType.getType());
+                }
+            }
+            return false;
+        }
+
+        /**
+         * Getter for codeJarType.
+         * 
+         * @return the codeJarType
+         */
+        public String getCodeJarType() {
+            return codeJarType;
+        }
+
+        /**
+         * Getter for codeType.
+         * 
+         * @return the codeType
+         */
+        public String getCodeType() {
+            return codeType;
+        }
+
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/repository/SVNConstant.java

@@ -27,6 +27,8 @@ public interface SVNConstant {
 
     public static final String NAME_MASTER = "master"; //$NON-NLS-1$
 
+    public static final String NAME_MAIN = "main"; //$NON-NLS-1$
+
     public static final String NAME_ORIGIN = "origin"; //$NON-NLS-1$
 
     /*

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/routines/CodesJarInfo.java

@@ -0,0 +1,169 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.model.routines;
+
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.eclipse.core.runtime.Assert;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.exception.PersistenceException;
+import org.talend.core.model.properties.Property;
+import org.talend.core.model.properties.RoutinesJarItem;
+import org.talend.core.model.repository.ERepositoryObjectType;
+import org.talend.core.model.repository.IRepositoryViewObject;
+import org.talend.core.runtime.repository.item.ItemProductKeys;
+import org.talend.cwm.helper.ResourceHelper;
+import org.talend.designer.core.model.utils.emf.component.IMPORTType;
+import org.talend.repository.ProjectManager;
+import org.talend.repository.model.IProxyRepositoryService;
+
+/**
+ * Could get most of codesjar's attribute in EMF model but some are lost when model is unloaded. Store those attributes
+ * in this bean.
+ */
+
+public class CodesJarInfo {
+
+    private static final String EMPTY_DATE;
+
+    private String projectTechName;
+
+    private String id;
+
+    private String label;
+
+    private String version;
+
+    private ERepositoryObjectType type;
+
+    private List<IMPORTType> imports;
+
+    private String modifiedDate;
+
+    static {
+        Calendar c = Calendar.getInstance();
+        c.setTimeInMillis(0);
+        EMPTY_DATE = ResourceHelper.dateFormat().format(c.getTime());
+    }
+
+    private CodesJarInfo() {
+        imports = new ArrayList<>();
+    }
+
+    public static CodesJarInfo create(Property property) {
+        Assert.isTrue(property.getItem() instanceof RoutinesJarItem);
+        CodesJarInfo info = new CodesJarInfo();
+        info.projectTechName = ProjectManager.getInstance().getProject(property).getTechnicalLabel();
+        info.id = property.getId();
+        info.label = property.getLabel();
+        info.version = property.getVersion();
+        info.type = ERepositoryObjectType.getItemType(property.getItem());
+        if (((RoutinesJarItem) property.getItem()).getRoutinesJarType() != null) {
+            info.imports.addAll(((RoutinesJarItem) property.getItem()).getRoutinesJarType().getImports());
+        }
+        String modifiedDate = (String) property.getAdditionalProperties().get(ItemProductKeys.DATE.getModifiedKey());
+        info.modifiedDate = StringUtils.isNotBlank(modifiedDate) ? modifiedDate : EMPTY_DATE;
+        return info;
+    }
+
+    public Property getProperty() {
+        try {
+            IRepositoryViewObject obj = IProxyRepositoryService.get().getProxyRepositoryFactory().getLastVersion(id);
+            if (obj != null) {
+                return obj.getProperty();
+            }
+        } catch (PersistenceException e) {
+            ExceptionHandler.process(e);
+        }
+        return null;
+     }
+
+    public String getProjectTechName() {
+        return projectTechName;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getLabel() {
+        return label;
+    }
+
+    public String getVersion() {
+        return version;
+    }
+
+    public ERepositoryObjectType getType() {
+        return type;
+    }
+
+    public List<IMPORTType> getImports() {
+        return imports;
+    }
+
+    public String getModifiedDate() {
+        return modifiedDate;
+    }
+
+    public boolean isInCurrentMainProject() {
+        return projectTechName.equals(ProjectManager.getInstance().getCurrentProject().getTechnicalLabel());
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((id == null) ? 0 : id.hashCode());
+        result = prime * result + ((label == null) ? 0 : label.hashCode());
+        result = prime * result + ((type == null) ? 0 : type.hashCode());
+        result = prime * result + ((version == null) ? 0 : version.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj)
+            return true;
+        if (obj == null)
+            return false;
+        if (getClass() != obj.getClass())
+            return false;
+        CodesJarInfo other = (CodesJarInfo) obj;
+        if (id == null) {
+            if (other.id != null)
+                return false;
+        } else if (!id.equals(other.id))
+            return false;
+        if (label == null) {
+            if (other.label != null)
+                return false;
+        } else if (!label.equals(other.label))
+            return false;
+        if (type == null) {
+            if (other.type != null)
+                return false;
+        } else if (!type.equals(other.type))
+            return false;
+        if (version == null) {
+            if (other.version != null)
+                return false;
+        } else if (!version.equals(other.version))
+            return false;
+        return true;
+    }
+
+}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/routines/IRoutinesService.java

@@ -12,11 +12,9 @@
 // ============================================================================
 package org.talend.core.model.routines;
 
-import java.util.Set;
 import java.util.Vector;
 
 import org.talend.core.IService;
-import org.talend.core.model.general.ModuleNeeded;
 
 /**
  *
@@ -26,5 +24,4 @@ public interface IRoutinesService extends IService {
 
     public Vector getAccents();
 
-    Set<ModuleNeeded> getRunningModules();
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/routines/RoutineLibraryMananger.java

@@ -120,8 +120,12 @@ public class RoutineLibraryMananger {
             initialized = true;
         }
     }
+    
+    public boolean needDeploy(URL fileUrl) throws IOException, Exception{
+        return needDeploy(fileUrl, null);
+    }
 
-    public boolean needDeploy(URL fileUrl) throws IOException, Exception {
+    public boolean needDeploy(URL fileUrl, String mvnUrl) throws IOException, Exception {
         File file = new File(fileUrl.getFile());
         ILibraryManagerService libManagerService = null;
         if (GlobalServiceRegister.getDefault().isServiceRegistered(ILibraryManagerService.class)) {
@@ -129,19 +133,21 @@ public class RoutineLibraryMananger {
                     .getService(ILibraryManagerService.class);
         }
         if (libManagerService != null) {
-            Map<String, String> sourceAndMavenUri = new HashMap<>();
-            libManagerService.guessMavenRUIFromIndex(file, sourceAndMavenUri);
-            String mavUri = null;
-            boolean isSnapshot = false;
-            for (String key : sourceAndMavenUri.keySet()) {
-                if (sourceAndMavenUri.get(key).equals(file.getPath())) {
-                    mavUri = key;
-                    break;
+            String mavUri = mvnUrl;
+            if (StringUtils.isBlank(mvnUrl)) {
+                Map<String, String> sourceAndMavenUri = new HashMap<>();
+                libManagerService.guessMavenRUIFromIndex(file, sourceAndMavenUri);
+                for (String key : sourceAndMavenUri.keySet()) {
+                    if (sourceAndMavenUri.get(key).equals(file.getPath())) {
+                        mavUri = key;
+                        break;
+                    }
+                }
+                if (mavUri == null) {
+                    return true;
                 }
             }
-            if (mavUri == null) {
-                return true;
-            }
+            boolean isSnapshot = false;
             final MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(mavUri);
             if (parseMvnUrl != null) {
                 if (parseMvnUrl.getVersion() != null && parseMvnUrl.getVersion().endsWith(MavenConstants.SNAPSHOT)) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/routines/RoutinesUtil.java

@@ -13,6 +13,7 @@
 package org.talend.core.model.routines;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 
@@ -20,12 +21,16 @@ import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IMarker;
 import org.eclipse.core.resources.IResource;
 import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.Path;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.exception.SystemException;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.model.general.Project;
+import org.talend.core.model.process.JobInfo;
 import org.talend.core.model.properties.Item;
+import org.talend.core.model.properties.JobletProcessItem;
+import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
 import org.talend.core.model.properties.RoutineItem;
 import org.talend.core.model.repository.ERepositoryObjectType;
@@ -204,6 +209,7 @@ public final class RoutinesUtil {
     }
 
     private static RoutinesParameterType createItemInforType(RoutineItem routineItem) {
+        // no need to update
         Property property = routineItem.getProperty();
 
         RoutinesParameterType itemRecordType = TalendFileFactory.eINSTANCE.createRoutinesParameterType();
@@ -353,4 +359,54 @@ public final class RoutinesUtil {
             }
         }
     }
+
+    @SuppressWarnings("unchecked")
+    public static void setInnerCodes(Property property, ERepositoryObjectType type) {
+        if (type == null) {
+            property.getAdditionalProperties().removeKey("JAR_TYPE");
+        } else {
+            property.getAdditionalProperties().put("JAR_TYPE", type.name()); //$NON-NLS-1$
+        }
+    }
+
+    public static ERepositoryObjectType getInnerCodeType(Property property) {
+        Object type = property.getAdditionalProperties().get("JAR_TYPE"); //$NON-NLS-1$
+        if (type != null) {
+            return ERepositoryObjectType.getAllTypesOfCodesJar().stream().filter(t -> t.name().equals(type)).findFirst().get();
+        }
+        return null;
+    }
+
+    public static boolean isInnerCodes(Property property) {
+        Object isInnerCodes = property.getAdditionalProperties().get("JAR_TYPE"); //$NON-NLS-1$
+        return isInnerCodes != null;
+    }
+
+    public static String getCodesJarLabelByInnerCode(Item innerCodeItem) {
+        return new Path(innerCodeItem.getState().getPath()).segment(0);
+    }
+
+    public static List<RoutinesParameterType> getRoutinesParametersFromJobInfo(JobInfo info) {
+        Item item = null;
+        if (info.getJobletProperty() != null) {
+            item = info.getJobletProperty().getItem();
+        } else if (info.getProcessItem() != null) {
+            item = info.getProcessItem();
+        }
+        return getRoutinesParametersFromItem(item);
+    }
+
+    public static List<RoutinesParameterType> getRoutinesParametersFromItem(Item item) {
+        List<RoutinesParameterType> routinesParameters = null;
+        if (item == null) {
+            routinesParameters = Collections.emptyList();
+        }
+        if (item instanceof JobletProcessItem) {
+            routinesParameters = ((JobletProcessItem) item).getJobletProcess().getParameters().getRoutinesParameter();
+        } else if (item instanceof ProcessItem) {
+            routinesParameters = ((ProcessItem) item).getProcess().getParameters().getRoutinesParameter();
+        }
+        return routinesParameters;
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/update/RepositoryUpdateManagerHelper.java

@@ -320,24 +320,28 @@ public class RepositoryUpdateManagerHelper {
 
         // all the jobs
         for (IRepositoryViewObject process : processRep) {
-            Item item = process.getProperty().getItem();
-            boolean found = false;
-            for (IProcess2 open : openedProcessList) {
-                if (open.getId().equals(item.getProperty().getId())) {
-                    found = true;
+            try {
+                Item item = process.getProperty().getItem();
+                boolean found = false;
+                for (IProcess2 open : openedProcessList) {
+                    if (open.getId().equals(item.getProperty().getId())) {
+                        found = true;
+                    }
                 }
-            }
-            if (found) {
-                continue;
-            }
-            checkMonitorCanceled(parentMonitor);
-            parentMonitor.subTask(RepositoryUpdateManager.getUpdateJobInfor(item.getProperty()));
+                if (found) {
+                    continue;
+                }
+                checkMonitorCanceled(parentMonitor);
+                parentMonitor.subTask(RepositoryUpdateManager.getUpdateJobInfor(item.getProperty()));
 
-            // List<UpdateResult> resultFromProcess = getResultFromProcess(process, types, onlySimpleShow);
+                // List<UpdateResult> resultFromProcess = getResultFromProcess(process, types, onlySimpleShow);
 
-            List<UpdateResult> resultFromProcess = getUpdatesNeededFromItems(parentMonitor, item, types);
-            if (resultFromProcess != null) {
-                resultList.addAll(resultFromProcess);
+                List<UpdateResult> resultFromProcess = getUpdatesNeededFromItems(parentMonitor, item, types);
+                if (resultFromProcess != null) {
+                    resultList.addAll(resultFromProcess);
+                }
+            } catch (Exception ex) {
+                ExceptionHandler.process(ex);
             }
             parentMonitor.worked(1);
         }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/ContextParameterUtils.java

@@ -207,6 +207,10 @@ public final class ContextParameterUtils {
     }
 
     private static ScriptEngine engine = new ScriptEngineManager().getEngineByName("JavaScript");
+    
+    public static ScriptEngine getScriptEngine() {
+        return engine;
+    }
 
     public static String convertContext2Literal4AnyVar(final String code, final IContext context) {
         if (code == null) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/NodeUtil.java

@@ -27,6 +27,9 @@ import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import javax.script.ScriptEngine;
+import javax.script.ScriptException;
+
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.model.components.ComponentCategory;
 import org.talend.core.model.components.IComponent;
@@ -46,7 +49,11 @@ import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.INode;
 import org.talend.core.model.process.INodeConnector;
 import org.talend.core.model.process.IProcess;
+import org.talend.core.model.process.IProcess2;
+import org.talend.core.model.process.ProcessUtils;
+import org.talend.core.model.properties.Property;
 import org.talend.core.runtime.IAdditionalInfo;
+import org.talend.core.runtime.projectsetting.RuntimeLineageManager;
 import org.talend.designer.core.ICamelDesignerCoreService;
 
 /**
@@ -866,6 +873,190 @@ public class NodeUtil {
         }
         throw new IllegalArgumentException();
     }
+    
+    
+    public static String getRuntimeParameterValue(INode node, IElementParameter ep) {
+        if (EParameterFieldType.TABLE.equals(ep.getFieldType())) {
+            Map<String, IElementParameter> types = new HashMap<String, IElementParameter>();
+            Object[] itemsValue = ep.getListItemsValue();
+            if (itemsValue != null) {
+                for (Object o : itemsValue) {
+                    IElementParameter cep = (IElementParameter) o;
+                    if (cep.isShow(node.getElementParameters())) {
+                        types.put(cep.getName(), cep);
+                    }
+                }
+            }
+            List<Map<String, String>> lines = (List<Map<String, String>>) ElementParameterParser.getObjectValue(node,
+                    "__" + ep.getName() + "__");
+            StringBuilder value = new StringBuilder();
+            // implement List & Map toString(), different is the value of Map
+            Iterator<Map<String, String>> linesIter = lines.iterator();
+            if (!linesIter.hasNext()) {
+                return "\"[]\"";
+            }
+            value.append("new StringBuilder().append(\"[");
+            for (;;) {
+                Map<String, String> columns = linesIter.next();
+                Iterator<Entry<String, String>> columnsIter = columns.entrySet().iterator();
+
+                value.append("{");
+                Entry<String, String> column = null;
+                boolean printedColumnExist = false;
+                while (columnsIter.hasNext()) {
+                    column = columnsIter.next();
+                    if (types.get(column.getKey()) == null) {
+                        continue;
+                    }
+                    printedColumnExist = true;
+
+                    value.append(column.getKey());
+                    value.append("=\").append(");
+                    value.append(getRuntimeParameterValue(column.getValue(), types.get(column.getKey()), true));
+                    value.append(").append(\"");
+
+                    if (columnsIter.hasNext()) {
+                        value.append(", ");
+                    }
+                }
+                if (printedColumnExist && column != null && (types.get(column.getKey()) == null)) {
+                    value.setLength(value.length() - 2);
+                }
+                value.append("}");
+
+                if (!linesIter.hasNext()) {
+                    return value.append("]\").toString()").toString();
+                }
+                value.append(",").append(" ");
+            }
+        } else {
+            String value = ElementParameterParser.getValue(node, "__" + ep.getName() + "__");
+            if (EParameterFieldType.TABLE_BY_ROW.equals(ep.getFieldType())) {
+                value = ep.getValue().toString();
+            }
+            return getRuntimeParameterValue(value, ep, false);
+        }
+    }
+    
+    private static String getRuntimeParameterValue(String value, IElementParameter ep, boolean itemFromTable) {
+        if (value == null) {
+            value = "";
+        }
+        
+        value = value.trim();
+        
+        boolean isMemo = false;
+        
+        List<EParameterFieldType> needRemoveCRLFList = Arrays.asList(EParameterFieldType.MEMO, EParameterFieldType.MEMO_JAVA,
+                EParameterFieldType.MEMO_SQL, EParameterFieldType.MEMO_IMPORT, EParameterFieldType.MEMO_MESSAGE);
+        if (needRemoveCRLFList.contains(ep.getFieldType())) {
+            isMemo = true;
+            value = value.replaceAll("[\r\n]", " ");
+        }
+        
+        List<EParameterFieldType> needQuoteList = Arrays.asList(EParameterFieldType.CLOSED_LIST,
+                EParameterFieldType.COMPONENT_LIST, EParameterFieldType.COLUMN_LIST, EParameterFieldType.PREV_COLUMN_LIST,
+                EParameterFieldType.CONNECTION_LIST, EParameterFieldType.LOOKUP_COLUMN_LIST,
+                EParameterFieldType.CONTEXT_PARAM_NAME_LIST, EParameterFieldType.PROCESS_TYPE, EParameterFieldType.COLOR,
+                EParameterFieldType.TABLE_BY_ROW, EParameterFieldType.HADOOP_JARS_DIALOG, EParameterFieldType.UNIFIED_COMPONENTS);
+        List<EParameterFieldType> needQuoteListForItem = itemFromTable ? Arrays.asList(EParameterFieldType.SCHEMA_TYPE,
+                EParameterFieldType.SAP_SCHEMA_TYPE, EParameterFieldType.MODULE_LIST) : new ArrayList<EParameterFieldType>();
+        // TODO: add RAW attribute when SCHEMA_COLUMN generated by BASED_ON_SCHEMA
+        List<String> needQuoteListByName = Arrays.asList("SCHEMA_COLUMN");// SCHEMA_COLUMN for BASED_ON_SCHEMA="true"
+
+        if (needQuoteList.contains(ep.getFieldType()) || needQuoteListForItem.contains(ep.getFieldType())
+                || needQuoteListByName.contains(ep.getName()) || ep.isRaw()) {
+            value = value.replaceAll("\\\\", "\\\\\\\\");
+            value = value.replaceAll("\\\"", "\\\\\\\"");
+            return "\"" + value + "\"";
+        }
+
+        if (itemFromTable) {
+            if ("*".equals(value)) {
+                return "\"" + value + "\"";
+            }
+            if (value.endsWith(";")) {
+                value = value.substring(0, value.length() - 1);
+            }
+        }
+        
+        if("".equals(value) || "\"\"".equals(value)) {
+            return "\"\"";
+        } else if("null".equals(value)) {
+            return "(Object)null";
+        }
+        
+        // copied it from Log4jFileUtil.javajet but need more comment for this script
+        if ("\"\\n\"".equals(value) || "\"\\r\"".equals(value) || "\"\\r\\n\"".equals(value)) {
+            // for the value is "\n" "\r" "\r\n"
+            return value.replaceAll("\\\\", "\\\\\\\\");
+        } else if ("\"\"\"".equals(value)) {
+            return "\"" + "\\" + "\"" + "\"";
+        } else if ("\"\"\\r\\n\"\"".equals(value)) {
+            return "\"\\\\r\\\\n\"";
+        } else if ("\"\"\\r\"\"".equals(value)) {
+            return "\"\\\\r\"";
+        } else if ("\"\"\\n\"\"".equals(value)) {
+            return "\"\\\\n\"";
+        }
+        // ftom 20141008 - patch to fix javajet compilation errors due to hard-coded studio TableEditor mechanism
+        // linked to BUILDIN properties checks, this item is a boolean set to TRUE or FALSE
+        // fix is just transforming into true or false to make logging OK
+        else if ("BUILDIN".equals(ep.getName())) {
+            return value.toLowerCase();
+        }
+        
+        //suppose all memo fields are processed well already, no need to go though this with dangerous
+        if (!isMemo && !org.talend.core.model.utils.ContextParameterUtils.isDynamic(value)) {
+            //https://jira.talendforge.org/browse/TDI-45563
+            //now can't get the var real value for "out1.A" in code generation time, so need to ignore it for runtime value log
+            //here consider the performance, only do it for some special case
+            if(itemFromTable && "VALUE".equals(ep.getName())) {
+                if(!isValidLiteralValue(value)) {
+                    return "\"\"";//return empty string as can't get the real runtime value
+                }
+            }
+            
+            if(value.length() > 1 && value.startsWith("\"") && value.endsWith("\"")) {
+                if(itemFromTable && "ARGS".equals(ep.getName())) {
+                    value = value.substring(1, value.length());
+                    value = value.substring(0, value.length() - 1);
+                    return "\"" + checkStringQuotationMarks(value) + "\"";
+                } else {
+                    //do nothing
+                    return value;
+                }
+            } else {
+                return "\"" + checkStringQuotationMarks(value) + "\"";
+            }
+        }
+        
+        //TODO remove it
+        if (value.endsWith("*")) {
+            return value.substring(0, value.length() - 1) + "\"*\"";
+        }
+        
+        return value;
+    }
+    
+    private static boolean isValidLiteralValue(String value) {
+        ScriptEngine se = ContextParameterUtils.getScriptEngine();
+        if(se==null) return true;
+        try {
+            se.eval(value);
+            return true;
+        } catch (ScriptException e) {
+            return false;
+        }
+    }
+    
+    private static String checkStringQuotationMarks(String str) {
+        String result = str;
+        if (result.contains("\"")) {
+            result = result.replace("\"", "\\\"");
+        }
+        return result;
+    }
 
     public static String getNormalizeParameterValue(INode node, IElementParameter ep) {
         if (EParameterFieldType.TABLE.equals(ep.getFieldType())) {
@@ -1189,7 +1380,7 @@ public class NodeUtil {
         } else if (ComponentCategory.CATEGORY_4_CAMEL.getName().equals(node.getComponent().getType())) {
             INodeConnector tmp = null;
             if (GlobalServiceRegister.getDefault().isServiceRegistered(ICamelDesignerCoreService.class)) {
-                ICamelDesignerCoreService camelService = (ICamelDesignerCoreService) GlobalServiceRegister.getDefault()
+                ICamelDesignerCoreService camelService = GlobalServiceRegister.getDefault()
                         .getService(ICamelDesignerCoreService.class);
                 tmp = node.getConnectorFromType(camelService.getTargetConnectionType(node));
             } else {
@@ -1232,4 +1423,37 @@ public class NodeUtil {
 
         return true;
     }
+
+    public static boolean isJobUsingRuntimeLineage(IProcess process) {
+        // Just support DI jobs now
+        boolean isSupport = isStandardJob(process) && !ProcessUtils.isTestContainer(process) && !isGuessSchemaJob(process);
+        if (!isSupport) {
+            return false;
+        }
+        RuntimeLineageManager runtimeLineageManager = new RuntimeLineageManager();
+        if (runtimeLineageManager.isUseRuntimeLineageAll()) {
+            return true;
+        }
+        if (runtimeLineageManager.getSelectedJobIds().isEmpty()) {
+            runtimeLineageManager.load();
+        }
+        return runtimeLineageManager.isRuntimeLineageSetting(process.getId());
+    }
+
+    public static boolean isStandardJob(IProcess process) {
+        if (process != null && process instanceof IProcess2) {
+            Property property = ((IProcess2) process).getProperty();
+            return property != null && property.getItem() != null
+                    && ComponentCategory.CATEGORY_4_DI.getName().equals(process.getComponentsType());
+        }
+        return false;
+    }
+
+    public static boolean isGuessSchemaJob(IProcess process) {
+        if (process != null && process instanceof IProcess2) {
+            Property property = ((IProcess2) process).getProperty();
+            return property != null && "ID".equals(property.getId()) && "Mock_job_for_Guess_schema".equals(property.getLabel()); //$NON-NLS-1$ //$NON-NLS-2$
+        }
+        return false;
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/UpdateRepositoryHelper.java

@@ -15,6 +15,7 @@ package org.talend.core.model.utils;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.talend.core.GlobalServiceRegister;
 import org.talend.core.model.metadata.builder.connection.Connection;
 import org.talend.core.model.metadata.builder.connection.DatabaseConnection;
 import org.talend.core.model.metadata.designerproperties.RepositoryToComponentProperty;
@@ -23,6 +24,7 @@ import org.talend.core.model.properties.ContextItem;
 import org.talend.core.model.properties.Item;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.update.UpdatesConstants;
+import org.talend.core.runtime.services.IGenericWizardService;
 
 /**
  * ggu class global comment. Detailled comment
@@ -47,6 +49,14 @@ public final class UpdateRepositoryHelper {
                 if (connection instanceof DatabaseConnection) {
                     String currentDbType = (String) RepositoryToComponentProperty.getValue(connection, UpdatesConstants.TYPE,
                             null);
+                    String productId = ((DatabaseConnection) connection).getProductId();
+                    if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericWizardService.class)) {
+                        IGenericWizardService service = GlobalServiceRegister.getDefault()
+                                .getService(IGenericWizardService.class);
+                        if (service != null && service.getIfAdditionalJDBCDBType(productId)) {
+                            currentDbType = productId;
+                        }
+                    }
                     aliasName += " (" + currentDbType + ")"; //$NON-NLS-1$ //$NON-NLS-2$
                 }
                 if (repositoryObjectType.getType().equals("SERVICES")) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/IRepositoryArtifactHandler.java

@@ -73,4 +73,6 @@ public interface IRepositoryArtifactHandler {
     
     public String resolveRemoteSha1(MavenArtifact artifact, boolean fromRelease) throws Exception;
 
+    public List<MavenArtifact> search(String name, boolean fromSnapshot) throws Exception;
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/NexusServerUtils.java

@@ -19,7 +19,9 @@ import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.http.HttpEntity;
@@ -53,6 +55,14 @@ public class NexusServerUtils {
     // the max search result is 200 by defult from nexus
     private static final int MAX_SEARCH_COUNT = 200;
 
+    public static final Set<String> IGNORED_TYPES = new HashSet<String>();
+
+    static {
+        IGNORED_TYPES.add("pom");
+        IGNORED_TYPES.add("sha1");
+        IGNORED_TYPES.add("md5");
+    }
+
     /**
      *
      * DOC check if the repository exist or not
@@ -236,36 +246,36 @@ public class NexusServerUtils {
     private static int readDocument(Document document, List<MavenArtifact> artifacts) throws Exception {
         List<Node> list = document.selectNodes("/searchNGResponse/data/artifact");//$NON-NLS-1$
         for (Node arNode : list) {
-            MavenArtifact artifact = new MavenArtifact();
-            artifacts.add(artifact);
-            artifact.setGroupId(arNode.selectSingleNode("groupId").getText());//$NON-NLS-1$
-            artifact.setArtifactId(arNode.selectSingleNode("artifactId").getText());//$NON-NLS-1$
-            artifact.setVersion(arNode.selectSingleNode("version").getText());//$NON-NLS-1$
-            Node descNode = arNode.selectSingleNode("description");//$NON-NLS-1$
-            if (descNode != null) {
-                artifact.setDescription(descNode.getText());
-            }
-            Node urlNode = arNode.selectSingleNode("url");//$NON-NLS-1$
-            if (urlNode != null) {
-                artifact.setUrl(urlNode.getText());
-            }
-            Node licenseNode = arNode.selectSingleNode("license");//$NON-NLS-1$
-            if (licenseNode != null) {
-                artifact.setLicense(licenseNode.getText());
-            }
-
-            Node licenseUrlNode = arNode.selectSingleNode("licenseUrl");//$NON-NLS-1$
-            if (licenseUrlNode != null) {
-                artifact.setLicenseUrl(licenseUrlNode.getText());
-            }
 
             List<Node> artLinks = arNode.selectNodes("artifactHits/artifactHit/artifactLinks/artifactLink");//$NON-NLS-1$
             for (Node link : artLinks) {
+                MavenArtifact artifact = new MavenArtifact();
+
+                artifact.setGroupId(arNode.selectSingleNode("groupId").getText());//$NON-NLS-1$
+                artifact.setArtifactId(arNode.selectSingleNode("artifactId").getText());//$NON-NLS-1$
+                artifact.setVersion(arNode.selectSingleNode("version").getText());//$NON-NLS-1$
+                Node descNode = arNode.selectSingleNode("description");//$NON-NLS-1$
+                if (descNode != null) {
+                    artifact.setDescription(descNode.getText());
+                }
+                Node urlNode = arNode.selectSingleNode("url");//$NON-NLS-1$
+                if (urlNode != null) {
+                    artifact.setUrl(urlNode.getText());
+                }
+                Node licenseNode = arNode.selectSingleNode("license");//$NON-NLS-1$
+                if (licenseNode != null) {
+                    artifact.setLicense(licenseNode.getText());
+                }
+
+                Node licenseUrlNode = arNode.selectSingleNode("licenseUrl");//$NON-NLS-1$
+                if (licenseUrlNode != null) {
+                    artifact.setLicenseUrl(licenseUrlNode.getText());
+                }
                 Node extensionElement = link.selectSingleNode("extension");//$NON-NLS-1$
                 String extension = null;
                 String classifier = null;
                 if (extensionElement != null) {
-                    if ("pom".equals(extensionElement.getText())) {//$NON-NLS-1$
+                    if (IGNORED_TYPES.contains(extensionElement.getText())) {// $NON-NLS-1$
                         continue;
                     }
                     extension = extensionElement.getText();
@@ -276,6 +286,7 @@ public class NexusServerUtils {
                 }
                 artifact.setType(extension);
                 artifact.setClassifier(classifier);
+                artifacts.add(artifact);
             }
         }
         return list.size();
@@ -370,4 +381,37 @@ public class NexusServerUtils {
 
     }
 
+    public static List<MavenArtifact> search(String nexusUrl, String userName, String password, String repositoryId, String name)
+            throws Exception {
+        List<MavenArtifact> artifacts = new ArrayList<MavenArtifact>();
+
+        int totalCount = 0;
+        String service = NexusConstants.SERVICES_SEARCH + getSearchQuery(repositoryId, null, null, null, 0, MAX_SEARCH_COUNT)
+                + "&q=" + name;
+
+        URI requestURI = getSearchURI(nexusUrl, service);
+        Document document = downloadDocument(requestURI, userName, password);
+        if (document != null) {
+            Node countNode = document.selectSingleNode("/searchNGResponse/totalCount");
+            if (countNode != null) {
+                try {
+                    totalCount = Integer.parseInt(countNode.getText());
+                } catch (NumberFormatException e) {
+                    totalCount = 0;
+                }
+            }
+            int searchDone = readDocument(document, artifacts);
+            while (searchDone < totalCount) {
+                service = NexusConstants.SERVICES_SEARCH
+                        + getSearchQuery(repositoryId, null, null, null, searchDone, MAX_SEARCH_COUNT) + "&q=" + name;
+                requestURI = getSearchURI(nexusUrl, service);
+
+                document = downloadDocument(requestURI, userName, password);
+                searchDone = searchDone + readDocument(document, artifacts);
+            }
+        }
+
+        return artifacts;
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/nexus/TalendLibsServerManager.java

@@ -231,7 +231,12 @@ public class TalendLibsServerManager {
         return canConnect;
     }
 
-    public ArtifactRepositoryBean getTalentArtifactServer() {
+    public boolean isProxyArtifactRepoConfigured() {
+        ArtifactRepositoryBean serverBean = getProxyArtifactServer();
+        return serverBean == null ? false : true;
+    }
+
+    public ArtifactRepositoryBean getProxyArtifactServer() {
         ArtifactRepositoryBean serverBean = new ArtifactRepositoryBean();
         // get from ini file first
         String url = System.getProperty(NEXUS_PROXY_URL);
@@ -255,9 +260,16 @@ public class TalendLibsServerManager {
                 serverBean.setType(prefManager.getValue(TalendLibsServerManager.NEXUS_PROXY_TYPE));
             }
         }
-        hasProxySetting = StringUtils.isNotEmpty(serverBean.getServer());
-        // use default
-        if (!hasProxySetting) {
+        if (StringUtils.isNotEmpty(serverBean.getServer())) {
+            return serverBean;
+        }
+        return null;
+    }
+
+    public ArtifactRepositoryBean getTalentArtifactServer() {
+        ArtifactRepositoryBean serverBean = getProxyArtifactServer();
+        if (serverBean == null) {
+            serverBean = new ArtifactRepositoryBean();
             serverBean.setServer(TALEND_LIB_SERVER);
             serverBean.setUserName(TALEND_LIB_USER);
             serverBean.setPassword(TALEND_LIB_PASSWORD);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/prefs/ITalendCorePrefConstants.java

@@ -114,7 +114,7 @@ public interface ITalendCorePrefConstants {
     public static final String DATA_COLLECTOR_PREVIEW = "data_collector_preview"; //$NON-NLS-1$
 
     public static final String DATA_COLLECTOR = "data_collector"; //$NON-NLS-1$
-
+    
     public static final String EXCHANGE_CHECK_TIS_VERSION = "exchange_check_tis_version"; //$NON-NLS-1$
 
     public static final String EXCHANGE_DOWNLOADED_CHECK_UPDATES = "exchange_downloaded_check_updates"; //$NON-NLS-1$
@@ -208,4 +208,8 @@ public interface ITalendCorePrefConstants {
 
     public static final int ARTIFACT_PROXY_SETTING = 1;
 
+    public static final String NEXUS_SHARE_LIBS = "NEXUS_SHARE_LIBS";
+
+    public static final boolean NEXUS_SHARE_LIBS_DEFAULT = false;
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/evaluator/AbstractPropertyValueEvaluator.java

@@ -15,6 +15,7 @@ package org.talend.core.runtime.evaluator;
 import java.util.List;
 
 import org.apache.commons.lang3.StringEscapeUtils;
+import org.talend.core.model.utils.ContextParameterUtils;
 import org.talend.core.runtime.util.GenericTypeUtils;
 import org.talend.core.utils.TalendQuoteUtils;
 import org.talend.daikon.properties.property.Property;
@@ -28,6 +29,10 @@ import org.talend.daikon.properties.property.StringProperty;
 public abstract class AbstractPropertyValueEvaluator implements PropertyValueEvaluator {
 
     public Object getTypedValue(Property property, Object rawValue) {
+        return getTypedValue(property, null, rawValue);
+    }
+
+    public Object getTypedValue(Property property, Object storedValue, Object rawValue) {
         if (GenericTypeUtils.isSchemaType(property)) {
             return rawValue;
         }
@@ -105,6 +110,12 @@ public abstract class AbstractPropertyValueEvaluator implements PropertyValueEva
                 String stringStoredValue = TalendQuoteUtils.removeQuotes(stringValue);
                 for (Object possibleValue : possibleValues) {
                     if (possibleValue.toString().equals(stringStoredValue)) {
+                        // Update since enum type set as context for tcompv0 .
+                        String currentStoredValue = String.valueOf(storedValue);
+                        if (storedValue != null && ContextParameterUtils.isContainContextParam(currentStoredValue)) {
+                            property.setTaggedValue("IS_CONTEXT_MODE", false);
+                            property.setValue(possibleValue);
+                        }
                         return possibleValue;
                     }
                 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/hd/IHDistribution.java

@@ -30,5 +30,7 @@ public interface IHDistribution {
     IHDistributionVersion getHDVersion(String v, boolean byDisplay);
 
     boolean useCustom();
+    
+    boolean isSparkLocal();
 
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenConstants.java

@@ -46,6 +46,8 @@ public interface MavenConstants {
 
     static final String EXCLUDE_DELETED_ITEMS = "EXCLUDE_DELETED_ITEMS";
 
+    static final String SKIP_FOLDERS = "SKIP_FOLDERS";
+
     /*
      * for lib
      */