APPINT-32905: Issue with Rest service flow

Co-authored-by: kjwang <kjwang@LT-B1Z2R73.talend.com>

i18n/plugins/org.talend.model.migration.nl/.classpath

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
-	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
-	<classpathentry kind="src" path="src"/>
-	<classpathentry kind="output" path="class"/>
-</classpath>

i18n/plugins/org.talend.model.migration.nl/.project

@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
-	<name>org.talend.model.migration.nl</name>
-	<comment></comment>
-	<projects>
-	</projects>
-	<buildSpec>
-		<buildCommand>
-			<name>org.eclipse.jdt.core.javabuilder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
-		<buildCommand>
-			<name>org.eclipse.pde.ManifestBuilder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
-		<buildCommand>
-			<name>org.eclipse.pde.SchemaBuilder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
-	</buildSpec>
-	<natures>
-		<nature>org.eclipse.pde.PluginNature</nature>
-		<nature>org.eclipse.jdt.core.javanature</nature>
-	</natures>
-</projectDescription>

i18n/plugins/org.talend.model.migration.nl/META-INF/MANIFEST.MF

@@ -1,6 +0,0 @@
-Manifest-Version: 1.0
-Bundle-ManifestVersion: 2
-Bundle-Name: Nl
-Bundle-SymbolicName: org.talend.model.migration.nl
-Bundle-Version: 7.4.1.qualifier
-Fragment-Host: org.talend.model.migration

i18n/plugins/org.talend.model.migration.nl/META-INF/eclipse.inf

@@ -1 +0,0 @@
-jarprocessor.exclude.children=true

i18n/plugins/org.talend.model.migration.nl/build.properties

@@ -1,13 +0,0 @@
-source.. = src/
-output.. = class/
-bin.includes = META-INF/,\
-               .\
-,plugin_ru.properties\
-,plugin_en.properties\
-,plugin_hr.properties\
-,plugin_fr.properties\
-,plugin_el.properties\
-,plugin_ja.properties\
-,plugin_it.properties\
-,plugin_de.properties\
-,plugin_swtbot.properties

i18n/plugins/org.talend.model.migration.nl/plugin_de.properties

@@ -1 +0,0 @@
-providerName=www.beispiel.de

i18n/plugins/org.talend.model.migration.nl/plugin_en.properties

@@ -1,2 +0,0 @@
-pluginName=Migration Model
-providerName=www.example.org

i18n/plugins/org.talend.model.migration.nl/plugin_fr.properties

@@ -1,2 +0,0 @@
-pluginName=Mod\u00E8le de migration
-providerName=www.example.org

i18n/plugins/org.talend.model.migration.nl/plugin_it.properties

@@ -1 +0,0 @@
-providerName=www.example.org

i18n/plugins/org.talend.model.migration.nl/plugin_ja.properties

@@ -1,2 +0,0 @@
-pluginName=\u79FB\u884C\u30E2\u30C7\u30EB
-providerName=www.example.org

i18n/plugins/org.talend.model.migration.nl/plugin_ru.properties

@@ -1 +0,0 @@
-providerName=www.example.org

i18n/plugins/org.talend.model.migration.nl/plugin_swtbot.properties

@@ -1,2 +0,0 @@
-pluginName = Migration Model
-providerName = www.example.org

i18n/plugins/org.talend.model.migration.nl/plugin_zh_CN.properties

@@ -1,2 +0,0 @@
-pluginName=\u8FC1\u79FB\u6A21\u578B
-providerName=www.example.org

i18n/plugins/org.talend.model.migration.nl/pom.xml

@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>org.talend.studio</groupId>
-    <artifactId>tcommon-studio-se</artifactId>
-    <version>7.4.1-SNAPSHOT</version>
-    <relativePath>../../../</relativePath>
-  </parent>
-  <artifactId>org.talend.model.migration.nl</artifactId>
-  <packaging>eclipse-plugin</packaging>
-</project>

i18n/plugins/org.talend.model.migration.nl/src/messages_en.properties

@@ -1,2 +0,0 @@
-TopMetadataMigrationFrom400to410usingGenericVM_top_error_log=error while loading the Transformation file : {0}
-TosMetadataMigrationFrom400to410_tos_error_log=error while loading the Transformation file : {0}

i18n/plugins/org.talend.model.migration.nl/src/messages_es.properties

@@ -1,2 +0,0 @@
-TopMetadataMigrationFrom400to410usingGenericVM_top_error_log=error mientras se cargaba el fichero de Transformaci\u00F3n: {0}
-TosMetadataMigrationFrom400to410_tos_error_log=error mientras se cargaba el fichero de Transformaci\u00F3n: {0}

i18n/plugins/org.talend.model.migration.nl/src/messages_fr.properties

@@ -1,2 +0,0 @@
-TopMetadataMigrationFrom400to410usingGenericVM_top_error_log=erreur lors du chargement du fichier de transformation : {0}
-TosMetadataMigrationFrom400to410_tos_error_log=Erreur lors du chargement du fichier de transformation : {0}

i18n/plugins/org.talend.model.migration.nl/src/messages_ja.properties

@@ -1,2 +0,0 @@
-TopMetadataMigrationFrom400to410usingGenericVM_top_error_log=\u5909\u63DB\u30D5\u30A1\u30A4\u30EB\u30ED\u30FC\u30C9\u4E2D\u306B\u30A8\u30E9\u30FC\uFF1A{0}
-TosMetadataMigrationFrom400to410_tos_error_log=\u5909\u63DB\u30D5\u30A1\u30A4\u30EB\u30ED\u30FC\u30C9\u4E2D\u306B\u30A8\u30E9\u30FC\uFF1A{0}

i18n/plugins/org.talend.model.migration.nl/src/messages_swtbot.properties

@@ -1,2 +0,0 @@
-TopMetadataMigrationFrom400to410usingGenericVM_top_error_log=error while loading the Transformation file : {0}
-TosMetadataMigrationFrom400to410_tos_error_log=error while loading the Transformation file : {0}

i18n/plugins/org.talend.model.migration.nl/src/messages_zh_CN.properties

@@ -1,2 +0,0 @@
-TopMetadataMigrationFrom400to410usingGenericVM_top_error_log=\u52A0\u8F7D\u8F6C\u6362\u6587\u4EF6\u65F6\u51FA\u9519\uFF1A{0}
-TosMetadataMigrationFrom400to410_tos_error_log=\u52A0\u8F7D\u8F6C\u6362\u6587\u4EF6\u65F6\u51FA\u9519\uFF1A{0}

main/features/org.talend.commons.feature/feature.xml

@@ -8,7 +8,6 @@
     <discovery label="Talend web site" url="http://www.talend.com"/> 
   </url>  
   <requires>
-    <import feature="org.talend.model.migration.feature" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.bouncycastle.bcprov" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="com.fasterxml.jackson.core.jackson-databind" version="2.9.8" match="greaterOrEqual"/>
     <import plugin="jackson-core-asl" version="0.0.0" match="greaterOrEqual"/>

main/features/org.talend.core.feature/feature.xml

@@ -38,7 +38,6 @@
     <import plugin="org.apache.httpcomponents.httpcore" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.eclipse.compare" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.eclipse.compare.core" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.m2m.atl.emftvm.trace" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.ops4j.pax.url.mvn" version="0.0.0" match="greaterOrEqual"/>
   </requires>  
   <plugin id="org.talend.core" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
@@ -66,8 +65,6 @@
   <plugin id="org.talend.model" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
   <plugin id="org.talend.model.edit" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
   <plugin id="org.talend.model.edit.nl" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
-  <plugin id="org.talend.model.migration" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
-  <plugin id="org.talend.model.migration.nl" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
   <plugin id="org.talend.model.nl" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
   <plugin id="org.talend.osgi.lib.loader" download-size="0" install-size="0" version="0.0.0" unpack="false" fragment="true"/>
   <plugin id="org.talend.resources" download-size="0" install-size="0" version="0.0.0"/>

main/features/org.talend.help.di.feature/.project

@@ -1,17 +1,17 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <projectDescription>
-	<name>org.talend.model.migration.feature</name>
+	<name>org.talend.help.di.feature</name>
 	<comment></comment>
 	<projects>
 	</projects>
 	<buildSpec>
 		<buildCommand>
-			<name>org.eclipse.pde.FeatureBuilder</name>
+			<name>org.eclipse.m2e.core.maven2Builder</name>
 			<arguments>
 			</arguments>
 		</buildCommand>
 	</buildSpec>
 	<natures>
-		<nature>org.eclipse.pde.FeatureNature</nature>
+		<nature>org.eclipse.m2e.core.maven2Nature</nature>
 	</natures>
 </projectDescription>

main/features/org.talend.jdt.feature/feature.xml

@@ -1,36 +1,55 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<feature
+      id="org.talend.jdt.feature"
+      label="Feature"
+      version="7.4.1.qualifier">
+
+   <description url="http://www.example.com/description">
+      [Enter Feature Description here.]
+   </description>
+
+   <copyright url="http://www.example.com/copyright">
+      [Enter Copyright Description here.]
+   </copyright>
+
+   <license url="http://www.example.com/license">
+      [Enter License Description here.]
+   </license>
+
+   <requires>
+      <import plugin="org.eclipse.jdt" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.annotation" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.apt.core" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.apt.pluggable.core" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.apt.ui" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.compiler.apt" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.compiler.tool" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.core" version="3.16.0.xx-201903290056-e1812-RELEASE" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.core.manipulation" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.debug" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.debug.ui" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.launching" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.ui" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.hamcrest.core" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.junit" version="4.13.2" match="greaterOrEqual"/>
+      <import plugin="org.eclipse.jdt.groovy.core" version="0.0.0" match="greaterOrEqual"/>
+      <import plugin="org.codehaus.groovy" version="2.4.16" match="equivalent"/>
+   </requires>
+
+   <plugin
+         id="org.eclipse.jdt.launching.macosx"
+         os="macosx"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
+   <plugin
+         id="org.eclipse.jdt.launching.ui.macosx"
+         os="macosx"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
 
-<feature id="org.talend.jdt.feature" label="Feature" version="7.4.1.qualifier"> 
-  <description url="http://www.example.com/description">[Enter Feature Description here.]</description>  
-  <copyright url="http://www.example.com/copyright">[Enter Copyright Description here.]</copyright>  
-  <license url="http://www.example.com/license">[Enter License Description here.]</license>  
-  <requires>
-    <!-- bonita requires org.codehaus.groovy version 2.4.16.xx-201903290056-e1812-RELEASE -->
-    <import plugin="org.codehaus.groovy" version="2.4.16.xx-201903290056-e1812-RELEASE" match="perfect"/>
-    <import plugin="org.eclipse.ant.launching" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.ant.ui" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.annotation" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.apt.core" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.apt.pluggable.core" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.apt.ui" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.compiler.apt" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.compiler.tool" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.core" version="3.16.0.xx-201903290056-e1812-RELEASE" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.core.manipulation" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.debug" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.debug.ui" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.doc.user" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.groovy.core" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.junit" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.junit.core" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.junit.runtime" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.junit4.runtime" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.launching" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.eclipse.jdt.ui" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.hamcrest.core" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.junit" version="0.0.0" match="greaterOrEqual"/>
-  </requires>
-  <plugin id="org.eclipse.jdt.launching.macosx" os="macosx" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
-  <plugin id="org.eclipse.jdt.launching.ui.macosx" os="macosx" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
 </feature>

main/features/org.talend.model.migration.feature/build.properties

@@ -1 +0,0 @@
-bin.includes = feature.xml

main/features/org.talend.model.migration.feature/feature.xml

@@ -1,26 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<feature
-      id="org.talend.model.migration.feature"
-      label="model Migration feature"
-      version="7.4.1.qualifier">
-
-   <description url="http://www.example.com/description">
-      [Enter Feature Description here.]
-   </description>
-
-   <copyright url="http://www.example.com/copyright">
-      [Enter Copyright Description here.]
-   </copyright>
-
-   <license url="http://www.example.com/license">
-      [Enter License Description here.]
-   </license>
-
-   <requires>
-      <import feature="org.eclipse.m2m.atl" version="0.0.0" match="greaterOrEqual"/>
-      <import feature="org.talend.utils.feature" version="0.0.0" match="greaterOrEqual"/>
-      <import plugin="org.bouncycastle.bcprov" version="0.0.0" match="greaterOrEqual"/>
-      <import plugin="org.objectweb.asm" version="0.0.0" match="greaterOrEqual"/>
-   </requires>
-
-</feature>

main/features/org.talend.model.migration.feature/pom.xml

@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>org.talend.studio</groupId>
-    <artifactId>tcommon-studio-se</artifactId>
-    <version>7.4.1-SNAPSHOT</version>
-    <relativePath>../../../</relativePath>
-  </parent>
-  <artifactId>org.talend.model.migration.feature</artifactId>
-  <packaging>eclipse-feature</packaging>
-</project>

main/features/org.talend.utils.feature/feature.xml

@@ -18,7 +18,7 @@
     <import plugin="org.apache.commons.lang3" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.apache.log4j" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.apache.servicemix.bundles.avro" version="0.0.0" match="greaterOrEqual"/>
-    <import plugin="org.junit" version="0.0.0" match="greaterOrEqual"/>
+    <import plugin="org.junit" version="4.13.2" match="greaterOrEqual"/>
     <import plugin="org.slf4j.api" version="0.0.0" match="greaterOrEqual"/>
     <import plugin="org.apache.commons.configuration" version="2.0.0" match="greaterOrEqual"/>
   </requires>  

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/VersionUtils.java

@@ -15,6 +15,7 @@ package org.talend.commons.utils;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.Optional;
@@ -352,6 +353,12 @@ public class VersionUtils {
         }
     }
 
+    public static File getStudioConfigFile() throws Exception {
+        URL configLocation = new URL("platform:/config/config.ini"); //$NON-NLS-1$
+        URL fileUrl = FileLocator.toFileURL(configLocation);
+        return URIUtil.toFile(new URI(fileUrl.getProtocol(), fileUrl.getPath(), fileUrl.getQuery()));
+    }
+
     public static String getSimplifiedPatchName(String projectPatchName) {
 
         if (projectPatchName != null) {

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/time/PropertiesCollectorUtil.java

@@ -0,0 +1,55 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.commons.utils.time;
+
+import java.io.File;
+import java.util.Properties;
+
+import org.eclipse.core.runtime.Platform;
+
+public class PropertiesCollectorUtil {
+
+    private static final String ROOT_NODE = "additional_packages_records"; // additional packages
+
+    private static final String PREF_ADDITIONAL_PACKAGES = "AdditionalPackages"; // preference node
+
+    private static final String ADDITONAL_PACKAGE_FILE = "additionalPackages";
+
+
+    public static String getAdditionalPackageRecording() {
+        File file = getRecordFile(ADDITONAL_PACKAGE_FILE);
+
+        Properties props = PropertiesFileUtil.read(file, true);
+        String records = props.getProperty(ROOT_NODE, "");
+
+        return records;
+    }
+
+    public static void storeAdditionalPackageRecording(String records) {
+        File file = getRecordFile(ADDITONAL_PACKAGE_FILE);
+        Properties props = PropertiesFileUtil.read(file, false);
+        props.setProperty(ROOT_NODE, records);
+        PropertiesFileUtil.store(file, props);
+    }
+
+    private static File getRecordFile(String fileName) {
+        String configurationLocation = Platform.getConfigurationLocation().getURL().getPath();
+        File file = new File(configurationLocation + "/data_collector/" + fileName);
+        return file;
+    }
+
+    public static String getAdditionalPackagePreferenceNode() {
+        return PREF_ADDITIONAL_PACKAGES;
+    }
+
+}

main/plugins/org.talend.core.runtime/mappings/mapping_Greenplum.xml

@@ -12,17 +12,26 @@
 			<dbType type="BPCHAR" ignoreLen="true" ignorePre="true" />
 			<dbType type="BYTEA" ignoreLen="true" ignorePre="true" />	
 			<dbType type="CHAR" defaultLength="50" ignorePre="true"/>
+			<dbType type="CHARACTER" defaultLength="50" ignorePre="true"/>
 			<dbType type="CID" ignoreLen="true" ignorePre="true" />
 			<dbType type="CIDR" ignoreLen="true" ignorePre="true" />
 			<dbType type="CIRCLE" ignoreLen="true" ignorePre="true" />
 			<dbType type="DATE" ignoreLen="true" ignorePre="true" />
 			<dbType type="FLOAT4" ignoreLen="true" ignorePre="true" />
+			<dbType type="REAL" ignoreLen="true" ignorePre="true" />
 			<dbType type="FLOAT8" ignoreLen="true" ignorePre="true" />
+			<dbType type="DOUBLE PRECISION" ignoreLen="true" ignorePre="true" />
 			<dbType type="INET" ignoreLen="true" ignorePre="true" />
 			<dbType type="INT2" ignoreLen="true" ignorePre="true" />
+			<dbType type="SMALLINT" ignoreLen="true" ignorePre="true" />
 			<dbType type="INT2VECTOR" ignoreLen="true" ignorePre="true" />
 			<dbType type="INT4" ignoreLen="true" ignorePre="true" />
 			<dbType type="INT8" ignoreLen="true" ignorePre="true" />
+			<dbType type="INTEGER" ignoreLen="true" ignorePre="true" />
+			<dbType type="CHARACTER VARYING" ignoreLen="true" ignorePre="true" />
+			<dbType type="BOOLEAN" ignoreLen="true" ignorePre="true" />
+			<dbType type="BIGINT" ignoreLen="true" ignorePre="true" />
+			<dbType type="BIT VARYING" ignoreLen="true" ignorePre="true" />
 			<dbType type="INTERVAL" ignorePre="true" />
 			<dbType type="LINE" ignoreLen="true" ignorePre="true" />
 			<dbType type="LSEG" ignoreLen="true" ignorePre="true" />
@@ -30,6 +39,7 @@
 			<dbType type="MONEY" ignoreLen="true" ignorePre="true" />	
 			<dbType type="NAME" ignoreLen="true" ignorePre="true" />
 			<dbType type="NUMERIC" defaultLength="20" defaultPrecision="10"/>
+			<dbType type="DECIMAL" defaultLength="20" defaultPrecision="10"/>
 			<dbType type="OID" ignoreLen="true" ignorePre="true"  />
 			<dbType type="OIDVECTOR" ignoreLen="true" ignorePre="true" />
 			<dbType type="PATH" ignoreLen="true" ignorePre="true" />
@@ -110,16 +120,19 @@
 			<talendToDbTypes><!-- Adviced mappings  -->
 				<talendType type="id_List"/>
 				<talendType type="id_Boolean">	
-					<dbType type="BOOL" default="true" />				
+					<dbType type="BOOL" default="true" />
+					<dbType type="BOOLEAN" />					
 				</talendType>
 				<talendType type="id_Byte">		
 				</talendType>
 				<talendType type="id_byte[]">
 					<dbType type="BIT" default="true" />
-					<dbType type="VARBIT" />				
+					<dbType type="VARBIT" />
+					<dbType type="BIT VARYING" />				
 				</talendType>
 				<talendType type="id_Character">
 					<dbType type="CHAR" default="true" />
+					<dbType type="CHARACTER" />
 					<dbType type="BPCHAR" />
 					<dbType type="VARCHAR" />
 					<dbType type="TEXT" />							
@@ -136,33 +149,47 @@
 					<dbType type="RELTIME" />						
 				</talendType>
 				<talendType type="id_BigDecimal">
-					<dbType type="NUMERIC" default="true" />							
+					<dbType type="NUMERIC" default="true" />
+					<dbType type="DECIMAL" />							
 					<dbType type="FLOAT4" />
+					<dbType type="REAL" />
 					<dbType type="FLOAT8"/>
+					<dbType type="DOUBLE PRECISION"/>
 				</talendType>
 				<talendType type="id_Double">
 					<dbType type="FLOAT4" />
+					<dbType type="REAL" />
 					<dbType type="FLOAT8" default="true" />
-					<dbType type="NUMERIC" />							
+					<dbType type="NUMERIC" />
+					<dbType type="DECIMAL" />	
+					<dbType type="DOUBLE PRECISION"/>						
 				</talendType>
 				<talendType type="id_Float">
 					<dbType type="FLOAT4" default="true" />
 					<dbType type="FLOAT8" />
-					<dbType type="NUMERIC" />				
+					<dbType type="REAL" />
+					<dbType type="NUMERIC" />
+					<dbType type="DECIMAL" />
+					<dbType type="DOUBLE PRECISION"/>				
 				</talendType>
 				<talendType type="id_Integer">
 					<dbType type="INT2" />
+					<dbType type="SMALLINT" />
 					<dbType type="INT4" default="true" />
 					<dbType type="SERIAL" />
 					<dbType type="INT8" />
-					<dbType type="BIGSERIAL" />										
+					<dbType type="BIGINT" />
+					<dbType type="BIGSERIAL" />
+					<dbType type="INTEGER" />										
 				</talendType>
 				<talendType type="id_Long">
 					<dbType type="INT2" />
 					<dbType type="INT4" />
 					<dbType type="SERIAL" />
 					<dbType type="INT8"  default="true" />
-					<dbType type="BIGSERIAL" />							
+					<dbType type="BIGINT" />
+					<dbType type="BIGSERIAL" />	
+					<dbType type="SMALLINT" />						
 				</talendType>
 				<talendType type="id_Object">
 					<dbType type="_POINT" />
@@ -209,18 +236,26 @@
 					<dbType type="_ABSTIME" />
 					<dbType type="_ACLITEM" />
 					<dbType type="_INT8" />
+					<dbType type="BIGINT" />
+					<dbType type="BIT VARYING" />
 					<dbType type="_VARBIT" />
 					<dbType type="_BIT" />			
 					<dbType type="_BOOL" />
+					<dbType type="BOOLEAN" />
 					<dbType type="BOX" />
 					<dbType type="_BOX" />
 					<dbType type="_BYTEA" />
 					<dbType type="_VARCHAR" />
+					<dbType type="CHARACTER VARYING" />
+					<dbType type="CHARACTER" />
+					<dbType type="INTEGER" />
+					<dbType type="REAL" />
 					<dbType type="_BPCHAR" />
 					<dbType type="_CID" />
 					<dbType type="_CIDR" />
 					<dbType type="_CIRCLE" />
 					<dbType type="_DATE" />
+					<dbType type="DOUBLE PRECISION"/>
 					<dbType type="_FLOAT8" />
 					<dbType type="_INET" />
 					<dbType type="INT2VECTOR" />
@@ -236,6 +271,7 @@
 					<dbType type="NAME" />
 					<dbType type="_NAME" />
 					<dbType type="_NUMERIC" />
+					<dbType type="DECIMAL" />
 					<dbType type="_OID" />
 					<dbType type="OIDVECTOR" />
 					<dbType type="_OIDVECTOR" />
@@ -247,26 +283,44 @@
 					<dbType type="INT4" />
 					<dbType type="SERIAL" />
 					<dbType type="INT8" />
-					<dbType type="BIGSERIAL" />					
+					<dbType type="BIGINT" />
+					<dbType type="BIGSERIAL" />
+					<dbType type="SMALLINT" />					
 				</talendType>
 				<talendType type="id_String">
 					<dbType type="BPCHAR" />
 					<dbType type="VARCHAR" default="true" />
+					<dbType type="CHARACTER VARYING" />
 					<dbType type="TEXT" />				
 				</talendType>
 			</talendToDbTypes>
 			<dbToTalendTypes>
 				<dbType type="INT2" >
 					<talendType type="id_Short" default="true" />				
+				</dbType>
+				<dbType type="SMALLINT" >
+					<talendType type="id_Short" default="true" />				
 				</dbType>				
 				<dbType type="INT4" >
 					<talendType type="id_Integer" default="true" />
-				</dbType>				
+				</dbType>
+				<dbType type="INTEGER" >
+     				<talendType type="id_Integer" default="true" />
+    			</dbType>
+    			<dbType type="CHARACTER VARYING" >
+					<talendType type="id_String" default="true" />				
+				</dbType>
+				<dbType type="BOOLEAN" >
+					<talendType type="id_Boolean" default="true" />				
+				</dbType> 				
 				<dbType type="SERIAL" >
 					<talendType type="id_Integer" default="true" />				
 				</dbType>				
 				<dbType type="INT8" >
 					<talendType type="id_Long" default="true" />				
+				</dbType>
+				<dbType type="BIGINT" >
+					<talendType type="id_Long" default="true" />				
 				</dbType>				
 				<dbType type="BIGSERIAL" >
 					<talendType type="id_Long" default="true" />				
@@ -274,14 +328,26 @@
 				<dbType type="FLOAT4" >
 					<talendType type="id_Float" default="true" />				
 					<talendType type="id_BigDecimal"/>			
-				</dbType>				
+				</dbType>
+				<dbType type="REAL" >
+					<talendType type="id_Float" default="true" />				
+					<talendType type="id_BigDecimal"/>			
+				</dbType>			
 				<dbType type="FLOAT8" >
 					<talendType type="id_Double" default="true" />				
 					<talendType type="id_BigDecimal"/>			
 				</dbType>
+				<dbType type="DOUBLE PRECISION" >
+					<talendType type="id_Double" default="true" />				
+					<talendType type="id_BigDecimal"/>			
+				</dbType>
 				<dbType type="NUMERIC">
 					<talendType type="id_Double"/>
 					<talendType type="id_BigDecimal" default="true" />			
+				</dbType>
+				<dbType type="DECIMAL">
+					<talendType type="id_Double"/>
+					<talendType type="id_BigDecimal" default="true" />			
 				</dbType>				
 				<dbType type="BOOL" >
 					<talendType type="id_Boolean" default="true" />				
@@ -291,9 +357,15 @@
 				</dbType>				
 				<dbType type="VARBIT" >
 					<talendType type="id_byte[]" default="true" />				
-				</dbType>							
+				</dbType>	
+				<dbType type="BIT VARYING" >
+					<talendType type="id_byte[]" default="true" />				
+				</dbType>						
 				<dbType type="CHAR" >
 					<talendType type="id_String" default="true" />				
+				</dbType>
+				<dbType type="CHARACTER" >
+					<talendType type="id_String" default="true" />				
 				</dbType>				
 				<dbType type="BPCHAR" >
 					<talendType type="id_String" default="true" />				

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/IMigrateDIMetadataItemService.java

@@ -1,34 +0,0 @@
-// ============================================================================
-//
-// Copyright (C) 2006-2021 Talend Inc. - www.talend.com
-//
-// This source code is available under agreement available at
-// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
-//
-// You should have received a copy of the agreement
-// along with this program; if not, write to Talend SA
-// 9 rue Pages 92150 Suresnes, France
-//
-// ============================================================================
-package org.talend.core;
-
-import org.talend.core.model.properties.Item;
-import org.talend.migration.IMigrationTask.ExecutionResult;
-
-/**
- * DOC bZhou class global comment. Detailled comment
- */
-public interface IMigrateDIMetadataItemService extends IService {
-
-    /**
-     * DOC bZhou Comment method "migrateDIItems".
-     *
-     * This method to invode the migration task in DI to do migration when import items from DQ perspective.
-     *
-     * It should be removed after merge the migrate mechanism.
-     *
-     * @param item
-     * @return
-     */
-    ExecutionResult migrateDIItems(Item item);
-}

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/PluginChecker.java

@@ -12,13 +12,19 @@
 // ============================================================================
 package org.talend.core;
 
+import java.io.BufferedReader;
+import java.io.File;
 import java.io.IOException;
 import java.net.URL;
+import java.nio.file.Files;
+import java.util.Properties;
 
 import org.apache.log4j.Logger;
 import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.Platform;
 import org.osgi.framework.Bundle;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.VersionUtils;
 
 /**
  * This class can check whether some specific plugins are loaded or not. <br/>
@@ -62,6 +68,8 @@ public class PluginChecker {
 
     private static final String PREVIEW_PLUGIN_ID = "org.talend.designer.component.preview"; //$NON-NLS-1$
 
+    private static final String REMOTE_PROVIDER_PLUGIN_ID = "org.talend.repository.remoteprovider"; //$NON-NLS-1$
+
     private static final String SVN_PROVIDER_PLUGIN_ID = "org.talend.repository.svnprovider"; //$NON-NLS-1$
 
     private static final String GIT_PROVIDER_PLUGIN_ID = "org.talend.repository.gitprovider"; //$NON-NLS-1$
@@ -140,6 +148,10 @@ public class PluginChecker {
 
     public static final String APACHE_CXF_PLUGIN_ID = "org.talend.libraries.apache.cxf"; //$NON-NLS-1$
 
+    public static final String SWTBOT_PLUGIN_ID = "org.eclipse.swtbot.eclipse.core"; //$NON-NLS-1$
+
+    private static Boolean isStudioLite;
+
     /**
      * Check if specific plug-in is loaded.
      *
@@ -158,6 +170,23 @@ public class PluginChecker {
         return isJobLetPluginLoaded();
     }
 
+    public static boolean isStudioLite() {
+        if (isStudioLite == null) {
+            try {
+                File studioConfigFile = VersionUtils.getStudioConfigFile();
+                Properties props = new Properties();
+                try (BufferedReader reader = Files.newBufferedReader(studioConfigFile.toPath())) {
+                    props.load(reader);
+                }
+                isStudioLite = Boolean.valueOf(props.getProperty("talend.studio.lite"));
+            } catch (Exception e) {
+                isStudioLite = false;
+                ExceptionHandler.process(e);
+            }
+        }
+        return isStudioLite;
+    }
+
     public static boolean isRefProjectLoaded() {
         return isPluginLoaded(REF_PROJECT_PLUGIN_ID);
     }
@@ -259,6 +288,10 @@ public class PluginChecker {
         return isPluginLoaded(PREVIEW_PLUGIN_ID);
     }
 
+    public static boolean isRemoteProviderPluginLoaded() {
+        return isPluginLoaded(REMOTE_PROVIDER_PLUGIN_ID);
+    }
+
     public static boolean isSVNProviderPluginLoaded() {
         return isPluginLoaded(SVN_PROVIDER_PLUGIN_ID);
     }
@@ -372,6 +405,10 @@ public class PluginChecker {
         return isPluginLoaded(BD_ROUTINE_PLUGIN_ID);
     }
 
+    public static boolean isSWTBotLoaded() {
+        return isPluginLoaded(SWTBOT_PLUGIN_ID);
+    }
+
     public static String getBundlePath(String bundleName) {
         String bundlePath = ""; //$NON-NLS-1$
         Bundle refBundle = Platform.getBundle(bundleName);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/EDatabase4DriverClassName.java

@@ -68,7 +68,7 @@ public enum EDatabase4DriverClassName {
     PSQL(EDatabaseTypeName.PSQL, "org.postgresql.Driver"), //$NON-NLS-1$
     PLUSPSQL(EDatabaseTypeName.PLUSPSQL, "org.postgresql.Driver"), //$NON-NLS-1$
     GREENPLUM(EDatabaseTypeName.GREENPLUM, "com.pivotal.jdbc.GreenplumDriver"), //$NON-NLS-1$
-
+    GREENPLUM_PSQL(EDatabaseTypeName.GREENPLUM, "org.postgresql.Driver"), //$NON-NLS-1$
     SAS(EDatabaseTypeName.SAS, "com.sas.rio.MVADriver"), //$NON-NLS-1$
     SAPHana(EDatabaseTypeName.SAPHana, "com.sap.db.jdbc.Driver"), //$NON-NLS-1$
     SQLITE(EDatabaseTypeName.SQLITE, "org.sqlite.JDBC"), //$NON-NLS-1$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/EDatabaseTypeName.java

@@ -336,10 +336,19 @@ public enum EDatabaseTypeName {
                             .getProjectPreferenceManager()
                             .getBoolean(IProjectSettingPreferenceConstants.METADATA_DBCONNECTION_ODBC_ENABLE);
             isSupport = isSupportODBC;
+        } else if (EDatabaseTypeName.SAS == this) {
+            isSupport = false;
         }
 
         return isSupport;
     }
+    
+    public static boolean isDeprecateDbType(String dbType) {
+    	if (EDatabaseTypeName.SAS.getXmlName().equals(dbType)) {
+            return true;
+        }
+    	return false;
+    }
 
     /**
      * Getter for dbType.

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/DatabaseConnStrUtil.java

@@ -114,6 +114,13 @@ public class DatabaseConnStrUtil {
                 && EDatabaseVersion4Drivers.SYBASEIQ_16_SA.getVersionValue().equals(dbVersion)) {
         	connStr = EDatabaseConnTemplate.SYBASEASE_16_SA;
         }
+        if (EDatabaseTypeName.GREENPLUM.getDisplayName().equals(dbType)) {
+            if (EDatabaseVersion4Drivers.GREENPLUM.getVersionValue().equals(dbVersion)) {
+                connStr = EDatabaseConnTemplate.GREENPLUM;
+            }else if(EDatabaseVersion4Drivers.GREENPLUM_PSQL.getVersionValue().equals(dbVersion)) {
+                connStr = EDatabaseConnTemplate.GREENPLUM_PSQL;
+            }
+        }
         EDatabaseVersion4Drivers version = EDatabaseVersion4Drivers.indexOfByVersion(dbVersion);
         if (connStr != null) {
             String s = connStr.getUrlTemplate(version);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/template/EDatabaseConnTemplate.java

@@ -56,6 +56,11 @@ public enum EDatabaseConnTemplate {
     GREENPLUM(new DbConnStr(EDatabaseTypeName.GREENPLUM, //
             "jdbc:pivotal:greenplum://<host>:<port>;;DatabaseName=<sid>;<property>", //$NON-NLS-1$
             "5432")), //$NON-NLS-1$
+    
+
+    GREENPLUM_PSQL(new DbConnStr(EDatabaseTypeName.GREENPLUM, //
+            "jdbc:postgresql://<host>:<port>/<sid>?<property>", //$NON-NLS-1$
+            "5432")), //$NON-NLS-1$
 
     ORACLEFORSID(new DbConnStr(EDatabaseTypeName.ORACLEFORSID, //
             "jdbc:oracle:thin:@<host>:<port>:<sid>", //$NON-NLS-1$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/version/EDatabaseVersion4Drivers.java

@@ -99,8 +99,8 @@ public enum EDatabaseVersion4Drivers {
     VERTICA_3(new DbVersion4Drivers(EDatabaseTypeName.VERTICA, "VERTICA 3 (Deprecated)", "VERTICA_3", "vertica_3.0_jdk_5.jar")), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
 
     // for bug 0017930
-
-    GREENPLUM(new DbVersion4Drivers(EDatabaseTypeName.GREENPLUM, "greenplum-5.1.4.000275.jar")), //$NON-NLS-1$
+    GREENPLUM_PSQL(new DbVersion4Drivers(EDatabaseTypeName.GREENPLUM,"PostgreSQL", "POSTGRESQL", "postgresql-8.4-703.jdbc4.jar")), //$NON-NLS-1$
+    GREENPLUM(new DbVersion4Drivers(EDatabaseTypeName.GREENPLUM,"Greenplum", "GREENPLUM", "greenplum-5.1.4.000275.jar")), //$NON-NLS-1$
 //    PSQL_V10(new DbVersion4Drivers(EDatabaseTypeName.PSQL, "v10", "V10", "postgresql-42.2.5.jar")),
     PSQL_V9_X(new DbVersion4Drivers(EDatabaseTypeName.PSQL, "v9 and later", "V9_X", "postgresql-42.2.14.jar")), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
     PSQL_PRIOR_TO_V9(new DbVersion4Drivers(EDatabaseTypeName.PSQL, "Prior to v9", "PRIOR_TO_V9", "postgresql-8.4-703.jdbc4.jar")), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/BigDataBasicUtil.java

@@ -53,7 +53,7 @@ public class BigDataBasicUtil {
 
     private static IDynamicDistributionManager getDynamicDistributionManager(IProgressMonitor monitor) throws Exception {
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IHadoopDistributionService.class)) {
-            IHadoopDistributionService hdService = (IHadoopDistributionService) GlobalServiceRegister.getDefault()
+            IHadoopDistributionService hdService = GlobalServiceRegister.getDefault()
                     .getService(IHadoopDistributionService.class);
             if (hdService != null) {
                 return hdService.getDynamicDistributionManager();

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/general/ModuleToInstall.java

@@ -191,6 +191,23 @@ public class ModuleToInstall {
         }
     }
 
+    @Override
+    public ModuleToInstall clone() {
+        ModuleToInstall module = new ModuleToInstall();
+        module.setName(this.name);
+        module.setRequired(this.required);
+        module.setContext(this.context);
+        module.setDescription(this.description);
+        module.setDistribution(this.distribution);
+        module.setFromCustomNexus(this.fromCustomNexus);
+        module.setLicenseType(this.licenseType);
+        module.setLicenseUrl(this.licenseUrl);
+        module.setMavenUri(this.mavenUri);
+        module.setUrl_description(this.url_description);
+        module.setUrl_download(this.url_download);
+        return module;
+    }
+
     public static String removeAuthenrizationInfo(String mavenUri) throws Exception {
         MavenArtifact ma = MavenUrlHelper.parseMvnUrl(mavenUri);
         ma.setUsername(""); //$NON-NLS-1$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataTalendType.java

@@ -32,6 +32,8 @@ import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.StringUtils;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.resources.IProject;
@@ -492,19 +494,55 @@ public final class MetadataTalendType {
         }
     }
 
+    private static String getSha1OfFile(File file) {
+        String sha1 = null;
+        FileInputStream fileInputStream = null;
+        try {
+            fileInputStream = new FileInputStream(file);
+            sha1 = DigestUtils.shaHex(fileInputStream);
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        } finally {
+            if (fileInputStream != null) {
+                try {
+                    fileInputStream.close();
+                } catch (IOException e) {
+                    e.printStackTrace(); // Just print the exception for debug.
+                }
+            }
+        }
+        return sha1;
+    }
+    
+    public static void copyFile(File in, IFile out) throws CoreException, IOException {
+        FileInputStream fis = new FileInputStream(in);
+        if (out.exists()) {
+            out.setContents(fis, true, false, null);
+        } else {
+            out.create(fis, true, null);
+        }
+        fis.close();
+    }
+
     public static void syncNewMappingFileToProject() throws SystemException {
         try {
             File sysMappingFiles = new File(MetadataTalendType.getSystemForderURLOfMappingsFile().getPath());
             IFolder projectMappingFolder = ResourceUtils.getProject(ProjectManager.getInstance().getCurrentProject())
                     .getFolder(MetadataTalendType.PROJECT_MAPPING_FOLDER);
             File projectMappingFiles = new File(projectMappingFolder.getLocationURI());
-            if (!sysMappingFiles.exists() || !projectMappingFiles.exists()
-                    || sysMappingFiles.list().length == projectMappingFiles.list().length) {
+            if (!sysMappingFiles.exists() || !projectMappingFiles.exists()) {
                 return;
             }
-
             for (File sysMapping : sysMappingFiles.listFiles()) {
                 IFile projectMapping = projectMappingFolder.getFile(sysMapping.getName());
+                if (projectMapping.exists() && StringUtils.equals(sysMapping.getName(), "mapping_Greenplum.xml")) {
+
+                    String sha1OfFile = DigestUtils.shaHex(projectMapping.getContents());
+                    String shalOfOldSystem = "8431f19215dacb3caa126778ae695954552cce2a";
+                    if (StringUtils.equals(sha1OfFile, shalOfOldSystem)) {
+                        copyFile(sysMapping, projectMapping);
+                    }
+                }
                 if (!projectMapping.exists()) {
                     FileInputStream fis = null;
                     try {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/MetadataToolAvroHelper.java

@@ -25,8 +25,10 @@ import org.apache.avro.SchemaBuilder.FieldAssembler;
 import org.apache.avro.SchemaBuilder.FieldBuilder;
 import org.apache.avro.SchemaBuilder.PropBuilder;
 import org.apache.avro.SchemaBuilder.RecordBuilder;
+import org.apache.avro.SchemaParseException;
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
 import org.eclipse.emf.common.util.EList;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.GlobalServiceRegister;
@@ -49,6 +51,10 @@ import orgomg.cwm.objectmodel.core.TaggedValue;
  */
 public final class MetadataToolAvroHelper {
 
+    private static Logger LOGGER = Logger.getLogger(MetadataToolAvroHelper.class);
+
+    private static final String COLUMN = "Column";
+
     private static final String TALEND_DB_COLUMN_NAME = "talend.field.dbColumnName";
     /**
      * @return An Avro schema with enriched properties from the incoming metadata table.
@@ -66,7 +72,7 @@ public final class MetadataToolAvroHelper {
                 dynamicPosition = i;
                 dynColumn = column;
             } else {
-                fa = convertToAvro(fa, column);
+                fa = convertToAvro(fa, column, i);
             }
             i++;
         }
@@ -137,7 +143,7 @@ public final class MetadataToolAvroHelper {
      * Build a field into a schema using enriched properties from the incoming column.
      */
     private static FieldAssembler<Schema> convertToAvro(FieldAssembler<Schema> fa,
-            org.talend.core.model.metadata.builder.connection.MetadataColumn in) {
+            org.talend.core.model.metadata.builder.connection.MetadataColumn in, int i) {
         ICoreService coreService = (ICoreService) GlobalServiceRegister.getDefault().getService(ICoreService.class);
         String label = in.getLabel();
         if (label != null && coreService != null) {
@@ -237,7 +243,27 @@ public final class MetadataToolAvroHelper {
         }
 
         type = in.isNullable() ? AvroUtils.wrapAsNullable(type) : type;
-        return defaultValue == null ? fb.type(type).noDefault() : fb.type(type).withDefault(defaultValue);
+        FieldAssembler<Schema> returnResult = null;
+        try {
+            if (defaultValue == null) {
+                returnResult = fb.type(type).noDefault();
+            } else {
+                returnResult = fb.type(type).withDefault(defaultValue);
+            }
+        } catch (SchemaParseException e) {
+            // if validation not pass from avro ,then generate a new name like Column0 to pass the validation, but
+            // actually the display name of schema and the column name is using TALEND_DB_COLUMN_NAME
+            String genColumn = COLUMN + i;
+            FieldBuilder<Schema> fbNew = fa.name(genColumn);
+            copyColumnProperties(fbNew, in);
+            if (defaultValue == null) {
+                returnResult = fbNew.type(type).noDefault();
+            } else {
+                returnResult = fbNew.type(type).withDefault(defaultValue);
+            }
+            LOGGER.info(e.getMessage() + ", use " + genColumn + " instead");
+        }
+        return returnResult;
     }
 
     private static Schema getLogicalTypeSchema(org.talend.core.model.metadata.builder.connection.MetadataColumn column) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/designerproperties/RepositoryToComponentProperty.java

@@ -1067,7 +1067,10 @@ public class RepositoryToComponentProperty {
                 return dbVersionString;
             } else if (EDatabaseTypeName.SYBASEASE.getDisplayName().equals(databaseType)) {
                 return dbVersionString;
-            } else {
+            } else if (EDatabaseTypeName.GREENPLUM.getDisplayName().equals(databaseType)) {
+                return dbVersionString;
+                
+            }else {
                 String driverValue = EDatabaseVersion4Drivers.getDriversStr(databaseType, dbVersionString);
                 if (EDatabaseConnTemplate.ORACLE_OCI.getDBDisplayName().equals(databaseType)
                         || EDatabaseConnTemplate.ORACLE_CUSTOM.getDBDisplayName().equals(databaseType)

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/migration/AbstractItemMigrationTask.java

@@ -146,7 +146,7 @@ public abstract class AbstractItemMigrationTask extends AbstractMigrationTask im
      * this returns all type handled by this migration task even the extended type that could benefit from this
      * migrations
      * */
-    List<ERepositoryObjectType> getAllTypes() {
+    protected List<ERepositoryObjectType> getAllTypes() {
         List<ERepositoryObjectType> declaredTypes = getTypes();
         ArrayList<ERepositoryObjectType> allTypes = new ArrayList<ERepositoryObjectType>(declaredTypes.size());
         allTypes.addAll(declaredTypes);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/EParameterFieldType.java

@@ -124,6 +124,7 @@ public enum EParameterFieldType {
     TACOKIT_BUTTON,
     TACOKIT_SUGGESTABLE_TABLE,
     TACOKIT_VALUE_SELECTION,
+    TACOKIT_TABLE,
     TACOKIT_TEXT_AREA_SELECTION;
 
     public String getName() {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/IProcess.java

@@ -12,6 +12,7 @@
 // ============================================================================
 package org.talend.core.model.process;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
@@ -133,4 +134,8 @@ public interface IProcess extends IElement {
     public String getComponentsType();
 
     public INode getNodeByUniqueName(String uniqueName);
+
+    default List<? extends INode> getProcessNodes() {
+        return new ArrayList<INode>();
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/JobInfo.java

@@ -67,6 +67,8 @@ public class JobInfo {
 
     private boolean needUnloadProcessor;
 
+    private int includeESBFlag = 0;
+
     public JobInfo(String jobId, String contextName, String version) {
         this.jobId = jobId;
         this.contextName = contextName;
@@ -572,4 +574,13 @@ public class JobInfo {
     public boolean isNeedUnloadProcessor() {
         return this.needUnloadProcessor;
     }
+
+    public int getIncludeESBFlag() {
+        return includeESBFlag;
+    }
+
+    public void setIncludeESBFlag(int includeESBFlag) {
+        this.includeESBFlag = includeESBFlag;
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/repository/RepositoryManager.java

@@ -119,7 +119,7 @@ public final class RepositoryManager {
     }
 
     public static void syncUserComponents() {
-        if (PluginChecker.isSVNProviderPluginLoaded()) {
+        if (PluginChecker.isRemoteProviderPluginLoaded()) {
             IComponentsService codeGenService = (IComponentsService) GlobalServiceRegister.getDefault().getService(
                     IComponentsService.class);
             if (codeGenService != null) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/NodeUtil.java

@@ -418,6 +418,52 @@ public class NodeUtil {
         return conns;
     }
 
+    /**
+     * DOC
+     * <p>
+     * The method searches for the incoming node connections of type
+     * on a processing path and returns the first ones only
+     * </p>
+     *
+     * @param node
+     * @param type - node type to look for
+     * @return
+             */
+    public static List<? extends IConnection> getFirstIncomingLineConnectionsOfType(INode node, String type) {
+        if (type == null)
+            return new ArrayList<IConnection>();
+
+        Set<String> uniqueNamesDone = new HashSet<String>();
+        List<? extends IConnection> allIncomingConnections = getFirstIncomingLineConnectionsOfType(node, uniqueNamesDone, type);
+
+        return allIncomingConnections;
+    }
+
+    private static List<? extends IConnection> getFirstIncomingLineConnectionsOfType(INode node, Set<String> uniqueNamesDone, String type) {
+        List<IConnection> conns = new ArrayList<IConnection>();
+
+        List<? extends IConnection> incomingConnections = node.getIncomingConnections();
+        if (incomingConnections != null) {
+
+            for (int i = 0; i < incomingConnections.size(); i++) {
+
+                IConnection connection = incomingConnections.get(i);
+                INode nextNode = connection.getSource();
+
+                if (!uniqueNamesDone.contains(nextNode.getUniqueName())) {
+                	uniqueNamesDone.add(nextNode.getUniqueName());
+
+                	if (type.equals((String)nextNode.getElementParameter("COMPONENT_NAME").getValue())) {
+                        conns.add(connection);
+                    } else {
+                        conns.addAll(getFirstIncomingLineConnectionsOfType(nextNode, uniqueNamesDone, type)); // follow this way
+                    }
+                }
+            }
+        }
+        return conns;
+    }
+
     public static INode getFirstMergeNode(INode node) {
         INode mergeNode = null;
         for (IConnection connection : node.getOutgoingConnections()) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/maven/MavenConstants.java

@@ -36,6 +36,8 @@ public interface MavenConstants {
 
     static final String DEFAULT_GROUP_ID = "org.talend";
 
+    static final String APACHE_GROUP_ID = "org.apache";
+
     static final String DEFAULT_VERSION = "6.0.0";
 
     static final String SNAPSHOT = "-SNAPSHOT";

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/process/TalendProcessOptionConstants.java

@@ -80,4 +80,13 @@ public interface TalendProcessOptionConstants {
      */
     public static final int MODULES_WITH_CODESJAR = 1 << 6;
 
+    /**
+     * flag for check if is ESB job
+     */
+    public static final int ISESB_CHECKED = 1;
+
+    public static final int ISESB_CHILDREN_INCLUDE = 1 << 1;
+
+    public static final int ISESB_CURRENT_INCLUDE = 1 << 2;
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/service/ICoreUIService.java

@@ -47,4 +47,6 @@ public interface ICoreUIService extends IService {
     String getPreferenceValue(String key);
 
     void loadComponentsFromProviders(ERepositoryObjectType type);
+    
+    void resetDataCollectorData();
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/IJobletProviderService.java

@@ -147,4 +147,6 @@ public interface IJobletProviderService extends IService {
 
     public void updateJobleModifiedRelated(Item item, String oldName, String newName);
 
+    public IProcess getJobletGEFProcessFromNode(INode node);
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/designer/runprocess/IRunProcessService.java

@@ -192,7 +192,7 @@ public interface IRunProcessService extends IService {
     @Deprecated
     public void buildJavaProject();
 
-    public void buildCodesJavaProject(IProgressMonitor monitor);
+    public void buildCodesJavaProject(IProgressMonitor monitor, Set<CodesJarInfo> toUpdate);
 
     /**
      * @deprecated use {@link IRunProcessService#getTalendJobJavaProject(Property)} instead

main/plugins/org.talend.core.runtime/src/main/java/org/talend/repository/ProjectManager.java

@@ -186,7 +186,7 @@ public final class ProjectManager {
             Set<String> resolvedProjectLabels, boolean force) {
         if (p != null) {
             String parentBranch = ProjectManager.getInstance().getMainProjectBranch(p);
-            if (parentBranch != null || force) {
+            if ((!p.isLocal() || parentBranch != null) || force) {
                 resolvedProjectLabels.add(p.getTechnicalLabel());
                 for (ProjectReference pr : new Project(p).getProjectReferenceList()) {
                     if (ProjectManager.validReferenceProject(p, pr)

main/plugins/org.talend.core.ui/plugin.xml

@@ -36,6 +36,12 @@
             id="org.talend.core.ui.token.RepositroyActionsTokenCollector"
             name="actions">
       </provider>
+      <provider
+            collector="org.talend.core.ui.token.AdditionalPackagesTokenCollector"
+            description="installed addtional package, old iems, amc"
+            id="org.talend.core.ui.token.AdditionalPackageTokenCollector"
+            name="addtional package">
+      </provider>
    </extension>
    
       <extension

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/CoreUIService.java

@@ -27,6 +27,7 @@ import org.talend.core.ui.component.ComponentPaletteUtilities;
 import org.talend.core.ui.component.ComponentsFactoryProvider;
 import org.talend.core.ui.services.IDesignerCoreUIService;
 import org.talend.core.ui.services.IRulesProviderService;
+import org.talend.core.ui.token.TokenCollectorFactory;
 
 /**
  * DOC ggu class global comment. Detailled comment
@@ -107,4 +108,9 @@ public class CoreUIService implements ICoreUIService {
             designerCoreUIService.loadComponentsFromProviders(type);
         }
     }
+    
+    @Override
+    public void resetDataCollectorData() {
+        TokenCollectorFactory.getFactory().reset();
+    }
 }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/token/AdditionalPackagesTokenCollector.java

@@ -0,0 +1,40 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.core.ui.token;
+
+import org.talend.commons.utils.time.PropertiesCollectorUtil;
+
+import us.monoid.json.JSONObject;
+
+public class AdditionalPackagesTokenCollector extends AbstractTokenCollector {
+
+    public AdditionalPackagesTokenCollector() {
+        // TODO Auto-generated constructor stub
+    }
+
+    @Override
+    public JSONObject collect() throws Exception {
+        String records = PropertiesCollectorUtil.getAdditionalPackageRecording();
+
+        JSONObject allRecords;
+        try {
+            allRecords = new JSONObject(records);
+        } catch (Exception e) {
+            // the value is not set, or is empty
+            allRecords = new JSONObject();
+            allRecords.put(PropertiesCollectorUtil.getAdditionalPackagePreferenceNode(), new JSONObject());
+        }
+
+        return allRecords;
+    }
+}

main/plugins/org.talend.core/src/main/java/org/talend/core/model/repository/ExternalNodesFactory.java

@@ -14,6 +14,7 @@ package org.talend.core.model.repository;
 
 import java.util.List;
 
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.IllegalPluginConfigurationException;
 import org.talend.commons.utils.workbench.extensions.ExtensionImplementationProvider;
 import org.talend.core.model.process.IExternalNode;
@@ -35,7 +36,8 @@ public class ExternalNodesFactory {
         try {
             listComponents = ExtensionImplementationProvider.getInstance(ExtensionPointFactory.EXTERNAL_COMPONENT, extensionId);
         } catch (IllegalPluginConfigurationException e) {
-            throw new RuntimeException("plugin:" + extensionId + " not found", e); //$NON-NLS-1$ //$NON-NLS-2$
+            ExceptionHandler.process(new Exception("plugin:" + extensionId + " not found", e)); //$NON-NLS-1$ //$NON-NLS-2$
+        	return null; //temporary
         }
         return listComponents.get(0);
     }

main/plugins/org.talend.core/src/main/java/org/talend/core/model/utils/ContextOrderProperties.java

@@ -32,7 +32,6 @@ import org.apache.commons.lang.StringUtils;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.core.i18n.Messages;
-import org.talend.core.model.context.ContextUtils;
 import org.talend.core.model.general.Project;
 import org.talend.core.model.process.IContextParameter;
 import org.talend.core.model.properties.Item;
@@ -79,6 +78,9 @@ public class ContextOrderProperties extends Properties {
     @Override
     public Object put(Object key, Object value) {
         keys.add(key);
+        if (value == null) {
+            return super.remove(key);
+        }
         return super.put(key, value);
     }
 

main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/CodeGeneratorRoutine.java

@@ -19,9 +19,11 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.eclipse.core.resources.IFolder;
 import org.eclipse.emf.common.util.EList;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
+import org.talend.commons.utils.workbench.resources.ResourceUtils;
 import org.talend.core.CorePlugin;
 import org.talend.core.language.ECodeLanguage;
 import org.talend.core.language.LanguageManager;
@@ -36,6 +38,7 @@ import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.RoutineItem;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
+import org.talend.core.model.routines.CodesJarInfo;
 import org.talend.core.runtime.services.IDesignerMavenService;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.core.utils.CodesJarResourceCache;
@@ -151,7 +154,8 @@ public final class CodeGeneratorRoutine {
                     }
                     if (routinesParameterTypes != null) {
                         routinesParameterTypes.stream().filter(r -> r.getType() != null)
-                                .map(r -> CodesJarResourceCache.getCodesJarById(r.getId())).filter(info -> info != null)
+                                .map(r -> CodesJarResourceCache.getCodesJarById(r.getId()))
+                                .filter(info -> info != null && hasInnerCodes(info))
                                 .forEach(info -> neededCodesJars.add(designerMavenService.getImportGAVPackageForCodesJar(info)));
                     }
                 });
@@ -160,6 +164,18 @@ public final class CodeGeneratorRoutine {
         return neededCodesJars;
     }
 
+    private static boolean hasInnerCodes(CodesJarInfo info) {
+        try {
+            IFolder folder = ResourceUtils
+                    .getProject(ProjectManager.getInstance().getProjectFromProjectTechLabel(info.getProjectTechName()))
+                    .getFolder(ERepositoryObjectType.getFolderName(info.getType())).getFolder(info.getLabel());
+            return folder.exists() && folder.members().length > 0;
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+        return false;
+    }
+
     /**
      * 
      * Deprecate for won't use pigudf any more

main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java

@@ -22,9 +22,12 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
@@ -64,8 +67,6 @@ import org.talend.core.language.LanguageManager;
 import org.talend.core.model.components.ComponentCategory;
 import org.talend.core.model.components.EComponentType;
 import org.talend.core.model.components.IComponent;
-import org.talend.core.model.components.IComponentsFactory;
-import org.talend.core.model.components.IComponentsService;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.Project;
 import org.talend.core.model.metadata.IMetadataColumn;
@@ -83,16 +84,16 @@ import org.talend.core.model.process.JobInfo;
 import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.process.ReplaceNodesInProcessProvider;
 import org.talend.core.model.properties.Item;
-import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
-import org.talend.core.model.relationship.Relation;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.repository.RepositoryManager;
 import org.talend.core.model.repository.job.JobResource;
 import org.talend.core.model.repository.job.JobResourceManager;
+import org.talend.core.model.routines.CodesJarInfo;
+import org.talend.core.model.routines.RoutinesUtil;
 import org.talend.core.model.utils.JavaResourcesHelper;
 import org.talend.core.repository.model.ProxyRepositoryFactory;
 import org.talend.core.runtime.CoreRuntimePlugin;
@@ -109,6 +110,7 @@ import org.talend.core.services.ISVNProviderService;
 import org.talend.core.ui.IJobletProviderService;
 import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.core.utils.BitwiseOptionUtils;
+import org.talend.core.utils.CodesJarResourceCache;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextParameterType;
 import org.talend.designer.core.model.utils.emf.talendfile.ContextType;
@@ -174,11 +176,11 @@ public class ProcessorUtilities {
 
     private static final Set<ModuleNeeded> retrievedJarsForCurrentBuild = new HashSet<ModuleNeeded>();
 
-    private static final Set<String> esbJobs = new HashSet<String>();
+    private static final Map<String, Integer> esbJobs = new HashMap<String, Integer>();
 
     private static boolean isDebug = false;
 
-    private static boolean isCIMode = false;
+    private static boolean isDynamicJobAndCITest = false;
 
     private static JobInfo mainJobInfo;
 
@@ -421,6 +423,76 @@ public class ProcessorUtilities {
         return false;
     }
 
+    public static boolean checkProcessLoopDependencies(IProcess mainProcess, String id, String version,
+            LinkedList<String> pathlink, Map<String, String> idToLatestVersion) {
+        if (ItemCacheManager.LATEST_VERSION.contains(version)) {
+            if (idToLatestVersion.get(id) == null) {
+                ProcessItem processItem = ItemCacheManager.getProcessItem(id);
+                version = processItem.getProperty().getVersion();
+                idToLatestVersion.put(id, version);
+            } else {
+                version = idToLatestVersion.get(id);
+            }
+        }
+        String pathNode = id + "-" + version;
+        if (pathlink.contains(pathNode)) {
+            return true;
+        }
+        pathlink.add(pathNode);
+
+        boolean hasLoop = false;
+        List<? extends INode> processNodes = mainProcess.getProcessNodes();
+        for (INode node : processNodes) {
+            if (!node.isActivate()) {
+                continue;
+            }
+
+            IElementParameter processIdParam = node.getElementParameter("PROCESS_TYPE_PROCESS");
+            if (processIdParam != null && StringUtils.isNotBlank((String) processIdParam.getValue())) {
+                String jobIds = (String) processIdParam.getValue();
+                String subNodeversion = (String) node.getElementParameter("PROCESS_TYPE_VERSION").getValue();
+                for (String jobId : jobIds.split(ProcessorUtilities.COMMA)) {
+                    if (StringUtils.isBlank(jobId)) {
+                        continue;
+                    }
+                    ProcessItem processItem = ItemCacheManager.getProcessItem(jobId, subNodeversion);
+                    IDesignerCoreService service = CorePlugin.getDefault().getDesignerCoreService();
+                    IProcess subProcess = service.getProcessFromProcessItem(processItem);
+                    hasLoop = checkProcessLoopDependencies(subProcess, jobId, subNodeversion, pathlink, idToLatestVersion);
+                    if (hasLoop) {
+                        break;
+                    }
+                }
+            } else {
+                if (GlobalServiceRegister.getDefault().isServiceRegistered(IJobletProviderService.class)) {
+                    IJobletProviderService jobletService = GlobalServiceRegister.getDefault()
+                            .getService(IJobletProviderService.class);
+                    if (jobletService != null) {
+                        IProcess jobletProcess = jobletService.getJobletGEFProcessFromNode(node);
+                        if (jobletProcess != null) {
+                            String jobletId = jobletProcess.getId();
+                            IElementParameter projectTecNameParam = jobletProcess.getElementParameter("PROJECT_TECHNICAL_NAME");
+                            if (projectTecNameParam != null && StringUtils.isNotBlank((String) projectTecNameParam.getValue())) {
+                                jobletId = projectTecNameParam.getValue() + ":" + jobletId;
+                            }
+                            hasLoop = checkProcessLoopDependencies(jobletProcess, jobletId, jobletProcess.getVersion(), pathlink,
+                                    idToLatestVersion);
+                        }
+                    }
+                }
+            }
+            if (hasLoop) {
+                break;
+            }
+        }
+
+        if (!hasLoop) {
+            pathlink.removeLast();
+        }
+
+        return hasLoop;
+    }
+
     private static IProcessor generateCode(IProcessor processor2, JobInfo jobInfo, String selectedContextName,
             boolean statistics, boolean trace, boolean needContext, int option, IProgressMonitor progressMonitor)
             throws ProcessorException {
@@ -520,11 +592,10 @@ public class ProcessorUtilities {
         jobInfo.setProcessor(processor);
 
         if (isMainJob && selectedProcessItem != null) {
-            Relation mainRelation = new Relation();
-            mainRelation.setId(jobInfo.getJobId());
-            mainRelation.setVersion(jobInfo.getJobVersion());
-            mainRelation.setType(RelationshipItemBuilder.JOB_RELATION);
-            hasLoopDependency = checkLoopDependencies(mainRelation, new HashMap<String, String>());
+            Property property = selectedProcessItem.getProperty();
+            String jobId = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel() + ":" + property.getId();
+            hasLoopDependency = checkProcessLoopDependencies(currentProcess, jobId, property.getVersion(),
+                    new LinkedList<String>(), new HashMap<String, String>());
             // clean the previous code in case it has deleted subjob
             cleanSourceFolder(progressMonitor, currentProcess, processor);
         }
@@ -627,233 +698,6 @@ public class ProcessorUtilities {
         return processor;
     }
 
-    public static boolean checkLoopDependencies(Relation mainJobInfo, Map<String, String> idToLastestVersionMap)
-            throws ProcessorException {
-        List<Relation> itemsJobRelatedTo = getItemsRelation(mainJobInfo, idToLastestVersionMap);
-        List<Relation> relationChecked = new ArrayList<>();
-        relationChecked.add(mainJobInfo);
-        return checkLoopDependencies(mainJobInfo, mainJobInfo, itemsJobRelatedTo, relationChecked, idToLastestVersionMap);
-    }
-
-    private static boolean checkLoopDependencies(Relation mainRelation, Relation currentRelation,
-            List<Relation> itemsJobRelatedTo,
-            List<Relation> relationChecked, Map<String, String> idToLastestVersionMap) throws ProcessorException {
-        boolean hasDependency = false;
-        for (Relation relation : itemsJobRelatedTo) {
-            try {
-                // means the tRunjob deactivate, or one of the specific version tRunjon deactivate, skip
-                Map<String, Set<String>> actTrunjobHM = getActivateTRunjobMap(currentRelation.getId(),
-                        currentRelation.getVersion());
-                if (actTrunjobHM.get(relation.getId()) == null
-                        || !actTrunjobHM.get(relation.getId()).contains(relation.getVersion())) {
-                    continue;
-                }
-            } catch (Exception e) {
-                throw new ProcessorException(e);
-            }
-
-            hasDependency = relation.getId().equals(mainRelation.getId())
-                    && relation.getVersion().equals(mainRelation.getVersion());
-            if (!hasDependency) {
-                List<Relation> itemsChildJob = getItemsRelation(relation, idToLastestVersionMap);
-                if (!relationChecked.contains(relation)) {
-                    relationChecked.add(relation);
-                    hasDependency = checkLoopDependencies(mainRelation, relation, itemsChildJob, relationChecked,
-                            idToLastestVersionMap);
-                }
-                if (!hasDependency) {
-                    for (Relation childRelation : itemsChildJob) {
-
-                        try {
-                            // means the tRunjob deactivate, or one of the specific version tRunjon deactivate, skip
-                            Map<String, Set<String>> activateTRunjobMap = getActivateTRunjobMap(relation.getId(),
-                                    relation.getVersion());
-                            if (activateTRunjobMap.get(childRelation.getId()) == null
-                                    || !activateTRunjobMap.get(childRelation.getId()).contains(childRelation.getVersion())) {
-                                continue;
-                            }
-                        } catch (Exception e) {
-                            throw new ProcessorException(e);
-                        }
-
-                        hasDependency = checkLoopDependencies(childRelation, idToLastestVersionMap);
-                        if (hasDependency) {
-                            break;
-                        }
-                    }
-                }
-            }
-            if (hasDependency) {
-                break;
-            }
-        }
-
-        return hasDependency;
-    }
-
-    private static Map<String, Set<String>> getActivateTRunjobMap(String id, String version) throws PersistenceException {
-        Map<String, Set<String>> actTrunjobHM = new HashMap<String, Set<String>>();
-        ProcessType processType = null;
-        try {
-            IRepositoryViewObject currentJobObject = ProxyRepositoryFactory.getInstance().getSpecificVersion(id, version, true);
-            if (currentJobObject != null) {
-                Item item = currentJobObject.getProperty().getItem();
-                if (item instanceof ProcessItem) {
-                    processType = ((ProcessItem) item).getProcess();
-                } else if (item instanceof JobletProcessItem) {
-                    processType = ((JobletProcessItem) item).getJobletProcess();
-                }
-            }
-        } catch (PersistenceException e) {
-            ExceptionHandler.process(e);
-        }
-        if (processType != null) {
-            List<Project> allProjects = new ArrayList<Project>();
-            allProjects.add(ProjectManager.getInstance().getCurrentProject());
-            allProjects.addAll(ProjectManager.getInstance().getAllReferencedProjects());
-
-            List<String> jobletsComponentsList = new ArrayList<String>();
-            IComponentsFactory componentsFactory = null;
-            if (GlobalServiceRegister.getDefault().isServiceRegistered(IComponentsService.class)) {
-                IComponentsService compService = GlobalServiceRegister.getDefault()
-                        .getService(IComponentsService.class);
-                if (compService != null) {
-                    componentsFactory = compService.getComponentsFactory();
-                    for (IComponent component : componentsFactory.readComponents()) {
-                        if (component.getComponentType() == EComponentType.JOBLET) {
-                            jobletsComponentsList.add(component.getName());
-                        }
-                    }
-                }
-            }
-
-            String jobletPaletteType = null;
-            String frameWork = processType.getFramework();
-            if (StringUtils.isBlank(frameWork)) {
-                jobletPaletteType = ComponentCategory.CATEGORY_4_DI.getName();
-            } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK)) {
-                jobletPaletteType = ComponentCategory.CATEGORY_4_SPARK.getName();
-            } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK_STREAMING)) {
-                jobletPaletteType = ComponentCategory.CATEGORY_4_SPARKSTREAMING.getName();
-            }
-
-            for (Object nodeObject : processType.getNode()) {
-                NodeType node = (NodeType) nodeObject;
-                // not tRunjob && not joblet then continue
-                if (!node.getComponentName().equals("tRunJob") && !jobletsComponentsList.contains(node.getComponentName())) { // $NON-NLS-1$
-                    continue;
-                }
-                boolean nodeActivate = true;
-                String processIds = null;
-                String processVersion = null;
-                for (Object elementParam : node.getElementParameter()) {
-                    ElementParameterType elemParamType = (ElementParameterType) elementParam;
-                    if ("PROCESS:PROCESS_TYPE_PROCESS".equals(elemParamType.getName())) { // $NON-NLS-1$
-                        processIds = elemParamType.getValue();
-                        if (StringUtils.isNotBlank(processIds)) {
-                            for (String jobId : processIds.split(ProcessorUtilities.COMMA)) {
-                                if (actTrunjobHM.get(jobId) == null) {
-                                    actTrunjobHM.put(jobId, new HashSet<String>());
-                                }
-                            }
-                        }
-                    } else if ("PROCESS:PROCESS_TYPE_VERSION".equals(elemParamType.getName()) // $NON-NLS-1$
-                            || "PROCESS_TYPE_VERSION".equals(elemParamType.getName())) { // $NON-NLS-1$
-                        processVersion = elemParamType.getValue();
-                    } else if ("ACTIVATE".equals(elemParamType.getName())) { // $NON-NLS-1$
-                        nodeActivate = Boolean.parseBoolean(elemParamType.getValue());
-                    }
-                }
-
-                if (nodeActivate) {
-                    if (StringUtils.isNotBlank(processIds)) {
-                        for (String jobId : processIds.split(ProcessorUtilities.COMMA)) {
-                            String actualVersion = processVersion;
-                            if (RelationshipItemBuilder.LATEST_VERSION.equals(processVersion)) {
-                                for (Project project : allProjects) {
-                                    IRepositoryViewObject lastVersion = null;
-                                    lastVersion = ProxyRepositoryFactory.getInstance().getLastVersion(project, jobId);
-                                    if (lastVersion != null) {
-                                        actualVersion = lastVersion.getVersion();
-                                        break;
-                                    }
-                                }
-                            }
-                            if (actTrunjobHM.get(jobId) != null) {
-                                actTrunjobHM.get(jobId).add(actualVersion);
-                            }
-
-                        }
-                    } else if (componentsFactory != null && jobletPaletteType != null) {
-                        // for joblet
-                        IComponent cc = componentsFactory.get(node.getComponentName(), jobletPaletteType);
-                        if (GlobalServiceRegister.getDefault().isServiceRegistered(IJobletProviderService.class)) {
-                            IJobletProviderService jobletService = GlobalServiceRegister.getDefault()
-                                    .getService(IJobletProviderService.class);
-                            Property property = jobletService.getJobletComponentItem(cc);
-                            if (property != null && StringUtils.isNotBlank(property.getId())) {
-                                String jobletId = property.getId();
-                                if (actTrunjobHM.get(jobletId) == null) {
-                                    actTrunjobHM.put(jobletId, new HashSet<String>());
-                                }
-                                String actualVersion = processVersion;
-                                if (RelationshipItemBuilder.LATEST_VERSION.equals(processVersion)) {
-                                    for (Project project : allProjects) {
-                                        IRepositoryViewObject lastVersion = null;
-                                        lastVersion = ProxyRepositoryFactory.getInstance().getLastVersion(project, jobletId);
-                                        if (lastVersion != null) {
-                                            actualVersion = lastVersion.getVersion();
-                                            break;
-                                        }
-                                    }
-                                }
-
-                                actTrunjobHM.get(jobletId).add(actualVersion);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        return actTrunjobHM;
-    }
-
-    private static List<Relation> getItemsRelation(Relation mainJobInfo, Map<String, String> idToLastestVersionMap) throws ProcessorException {
-        List<Relation> itemsJobRelatedTo = new ArrayList<Relation>();
-        try {
-            List<Project> allProjects = new ArrayList<Project>();
-            allProjects.add(ProjectManager.getInstance().getCurrentProject());
-            allProjects.addAll(ProjectManager.getInstance().getAllReferencedProjects());
-            RelationshipItemBuilder instance = RelationshipItemBuilder.getInstance();
-            if (instance != null) {
-                itemsJobRelatedTo.addAll(instance.getItemsChildRelatedTo(mainJobInfo.getId(), mainJobInfo.getVersion(),
-                        mainJobInfo.getType(), RelationshipItemBuilder.JOB_RELATION));
-                itemsJobRelatedTo.addAll(instance.getItemsChildRelatedTo(mainJobInfo.getId(), mainJobInfo.getVersion(),
-                        mainJobInfo.getType(), RelationshipItemBuilder.JOBLET_RELATION));
-                for (Relation relation : itemsJobRelatedTo) {
-                    if (relation.getVersion().equals(RelationshipItemBuilder.LATEST_VERSION)) {
-                        if (idToLastestVersionMap.containsKey(relation.getId())) {
-                            relation.setVersion(idToLastestVersionMap.get(relation.getId()));
-                        } else {
-                            for (Project project : allProjects) {
-                                IRepositoryViewObject lastVersion =
-                                        ProxyRepositoryFactory.getInstance().getLastVersion(project, relation.getId());
-                                if (lastVersion != null) {
-                                    relation.setVersion(lastVersion.getVersion());
-                                    idToLastestVersionMap.put(relation.getId(), relation.getVersion());
-                                    break;
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        } catch (PersistenceException e) {
-            throw new ProcessorException(e);
-        }
-
-        return itemsJobRelatedTo;
-    }
 
     private static void setNeededResources(final Map<String, Object> argumentsMap, JobInfo jobInfo) {
         argumentsMap.put(TalendProcessArgumentConstant.ARG_NEED_XMLMAPPINGS,
@@ -1099,8 +943,8 @@ public class ProcessorUtilities {
     private static IProcessor generateCode(JobInfo jobInfo, String selectedContextName, boolean statistics,
             boolean trace, boolean needContext, int option, IProgressMonitor progressMonitor)
             throws ProcessorException {
-        if (!BitwiseOptionUtils.containOption(option, GENERATE_WITHOUT_COMPILING)) {
-            CorePlugin.getDefault().getRunProcessService().buildCodesJavaProject(progressMonitor);
+        if (!BitwiseOptionUtils.containOption(option, GENERATE_WITHOUT_COMPILING) && jobInfo.getFatherJobInfo() == null) {
+            buildCodesJavaProject(progressMonitor, jobInfo);
         }
         return generateCode(jobInfo, selectedContextName, statistics, trace, needContext, true, option,
                 progressMonitor);
@@ -1202,11 +1046,9 @@ public class ProcessorUtilities {
             }
 
             if (isMainJob && selectedProcessItem != null) {
-                Relation mainRelation = new Relation();
-                mainRelation.setId(jobInfo.getJobId());
-                mainRelation.setVersion(jobInfo.getJobVersion());
-                mainRelation.setType(RelationshipItemBuilder.JOB_RELATION);
-                hasLoopDependency = checkLoopDependencies(mainRelation, new HashMap<String, String>());
+                Property property = selectedProcessItem.getProperty();
+                hasLoopDependency = checkProcessLoopDependencies(currentProcess, property.getId(), property.getVersion(),
+                        new LinkedList<String>(), new HashMap<String, String>());
                 // clean the previous code in case it has deleted subjob
                 cleanSourceFolder(progressMonitor, currentProcess, processor);
             }
@@ -1252,7 +1094,7 @@ public class ProcessorUtilities {
             checkMetadataDynamic(currentProcess, jobInfo);
 
             int options = TalendProcessOptionConstants.MODULES_DEFAULT;
-            if (isCIMode && BitwiseOptionUtils.containOption(option, GENERATE_MAIN_ONLY)) {
+            if (isCIMode() && BitwiseOptionUtils.containOption(option, GENERATE_MAIN_ONLY)) {
                 options |= TalendProcessOptionConstants.MODULES_WITH_CHILDREN;
             }
             Set<ModuleNeeded> neededLibraries = new HashSet<>();
@@ -1703,13 +1545,16 @@ public class ProcessorUtilities {
                         }
                     }
                 }
-                if (isEsbComponentName(componentName)) {
-                    addEsbJob(jobInfo);
-                }
             }
         }
     }
 
+    private static void recordESBIncludingFlag(JobInfo jobInfo, int esbIncludingOption) {
+        int includeESBFlag = jobInfo.getIncludeESBFlag();
+        includeESBFlag |= esbIncludingOption;
+        jobInfo.setIncludeESBFlag(includeESBFlag);
+    }
+
     static void setGenerationInfoWithChildrenJob(INode node, JobInfo jobInfo, final JobInfo subJobInfo) {
         final LastGenerationInfo generationInfo = LastGenerationInfo.getInstance();
 
@@ -1818,6 +1663,13 @@ public class ProcessorUtilities {
 
     static boolean hasLoopDependency = false;
 
+    private static void resetBuildFlagsAndCaches() {
+        jobList.clear();
+        esbJobs.clear();
+        hasLoopDependency = false;
+        mainJobInfo = null;
+    }
+
     /**
      * This function will generate the code of the process and all of this sub process.
      *
@@ -1832,15 +1684,10 @@ public class ProcessorUtilities {
         if (monitors == null) {
             monitor = new NullProgressMonitor();
         }
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         JobInfo jobInfo = new JobInfo(processName, contextName, version);
         IProcessor process = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return process;
     }
 
@@ -1860,14 +1707,9 @@ public class ProcessorUtilities {
         }
         JobInfo jobInfo = new JobInfo(processId, contextName, version);
         jobInfo.setApplyContextToChildren(applyContextToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor process = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return process;
     }
 
@@ -1882,14 +1724,9 @@ public class ProcessorUtilities {
         }
         JobInfo jobInfo = new JobInfo(process, contextName);
         jobInfo.setApplyContextToChildren(applyContextToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor result = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return result;
     }
 
@@ -1904,15 +1741,10 @@ public class ProcessorUtilities {
         }
         JobInfo jobInfo = new JobInfo(process, contextName, version);
         jobInfo.setApplyContextToChildren(applyContextToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor result =
                 generateCode(jobInfo, contextName, statistics, trace, needContext, GENERATE_ALL_CHILDS, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return result;
     }
 
@@ -1936,14 +1768,9 @@ public class ProcessorUtilities {
                 ProcessUtils.isOptionChecked(argumentsMap, TalendProcessArgumentConstant.ARG_NEED_CONTEXT);
         int option = ProcessUtils.getOptionValue(argumentsMap, TalendProcessArgumentConstant.ARG_GENERATE_OPTION, 0);
 
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor result = generateCode(jobInfo, contextName, statistics, trace, needContext, option, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return result;
     }
 
@@ -1961,14 +1788,9 @@ public class ProcessorUtilities {
             JobInfo jobInfo = new JobInfo(process, contextName, version);
             jobInfo.setContext(context);
             jobInfo.setApplyContextToChildren(applyContextToChildren);
-            jobList.clear();
-            esbJobs.clear();
-            hasLoopDependency = false;
-            mainJobInfo = null;
+            resetBuildFlagsAndCaches();
             result = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
-            jobList.clear();
-            hasLoopDependency = false;
-            mainJobInfo = null;
+            resetBuildFlagsAndCaches();
         }
         return result;
     }
@@ -2007,14 +1829,9 @@ public class ProcessorUtilities {
 
     public static IProcessor generateCode(ProcessItem process, String contextName, boolean statistics, boolean trace)
             throws ProcessorException {
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor returnValue = generateCode(process, contextName, statistics, trace, false);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return returnValue;
     }
 
@@ -2036,29 +1853,19 @@ public class ProcessorUtilities {
             jobInfo = new JobInfo(process, context);
         }
         jobInfo.setApplyContextToChildren(applyToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor genCode = generateCode(jobInfo, context.getName(), statistics, trace, contextProperties,
                 GENERATE_ALL_CHILDS, new NullProgressMonitor());
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return genCode;
     }
 
     public static IProcessor generateCode(IProcess process, IContext context, boolean statistics, boolean trace,
             boolean properties) throws ProcessorException {
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor returnValue =
                 generateCode(process, context, statistics, trace, properties, new NullProgressMonitor());
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return returnValue;
     }
 
@@ -2079,15 +1886,10 @@ public class ProcessorUtilities {
         } else {
             jobInfo = new JobInfo(process, context);
         }
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor genCode = generateCode(jobInfo, context.getName(), statistics, trace, properties,
                 GENERATE_ALL_CHILDS, progressMonitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return genCode;
     }
 
@@ -2096,8 +1898,6 @@ public class ProcessorUtilities {
 
         updateCodeSources();
 
-        CorePlugin.getDefault().getRunProcessService().buildCodesJavaProject(progressMonitor);
-
         // achen modify to fix 0006107
         ProcessItem pItem = null;
 
@@ -2115,6 +1915,9 @@ public class ProcessorUtilities {
         } else {
             jobInfo = new JobInfo(process, context);
         }
+
+        buildCodesJavaProject(progressMonitor, jobInfo);
+
         final boolean oldMeasureActived = TimeMeasure.measureActive;
         if (!oldMeasureActived) { // not active before.
             TimeMeasure.display = TimeMeasure.displaySteps = TimeMeasure.measureActive = CommonsPlugin.isDebugMode();
@@ -2123,15 +1926,10 @@ public class ProcessorUtilities {
                 + (jobInfo.getJobName() != null ? jobInfo.getJobName() : jobInfo.getJobId());
         TimeMeasure.begin(timeMeasureGenerateCodesId);
 
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor genCode = generateCode(processor, jobInfo, context.getName(), statistics, trace, properties,
                 GENERATE_ALL_CHILDS, progressMonitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
 
         TimeMeasure.end(timeMeasureGenerateCodesId);
         // if active before, not disable and active still.
@@ -2147,18 +1945,23 @@ public class ProcessorUtilities {
         updateCodeSources();
         // achen modify to fix 0006107
         JobInfo jobInfo = new JobInfo(process, context);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor genCode = generateCode(jobInfo, context.getName(), statistics, trace, properties, option,
                 new NullProgressMonitor());
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return genCode;
     }
 
+    private static void buildCodesJavaProject(IProgressMonitor monitor, JobInfo jobInfo) {
+        Set<JobInfo> allJobInfo = getChildrenJobInfo(jobInfo.getProcessItem(), false, true);
+        allJobInfo.add(jobInfo);
+        Set<CodesJarInfo> toUpdate = allJobInfo.stream().filter(childInfo -> !childInfo.isTestContainer())
+                .flatMap(childInfo -> RoutinesUtil.getRoutinesParametersFromJobInfo(childInfo).stream())
+                .filter(r -> r.getType() != null).map(r -> CodesJarResourceCache.getCodesJarById(r.getId()))
+                .filter(info -> info != null).collect(Collectors.toSet());
+        IRunProcessService.get().buildCodesJavaProject(monitor, toUpdate);
+    }
+
     /**
      *
      * @deprecated seems never use this one
@@ -2461,8 +2264,15 @@ public class ProcessorUtilities {
                     }
                     JobInfo jobInfo = new JobInfo(testItem, testProcess.getDefaultContext());
                     jobInfo.setTestContainer(true);
-                    jobInfos.add(jobInfo);
                     jobInfo.setFatherJobInfo(parentJobInfo);
+                    if (!jobInfos.contains(jobInfo)) {
+                        jobInfos.add(jobInfo);
+
+                        // if job contains testcase, we need to get joblets of testcase and add them to the job pom, we
+                        // must
+                        // pass parentJobInfo instead of testProcess, otherwise joblet will be filtered out
+                        getSubjobInfo(testProcess.getNode(), testProcess, parentJobInfo, jobInfos, firstChildOnly, includeJoblet);
+                    }
                 }
             }
         }
@@ -2480,6 +2290,8 @@ public class ProcessorUtilities {
         } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK_STREAMING)) {
             jobletPaletteType = ComponentCategory.CATEGORY_4_SPARKSTREAMING.getName();
         }
+
+        boolean hasChildrenIncludeESB = false;
         for (NodeType node : nodes) {
             boolean activate = true;
             // check if node is active at least.
@@ -2494,8 +2306,8 @@ public class ProcessorUtilities {
                 continue;
             }
 
-            if (isEsbComponentName(node.getComponentName())) {
-                addEsbJob(parentJobInfo);
+            if (!firstChildOnly && isEsbComponentName(node.getComponentName())) {
+                recordESBIncludingFlag(parentJobInfo, TalendProcessOptionConstants.ISESB_CURRENT_INCLUDE);
             }
 
             boolean isCTalendJob = "cTalendJob".equalsIgnoreCase(node.getComponentName());
@@ -2524,6 +2336,18 @@ public class ProcessorUtilities {
                                 jobInfo.setFatherJobInfo(parentJobInfo);
                                 if (!firstChildOnly) {
                                     getAllJobInfo(processItem.getProcess(), jobInfo, jobInfos, firstChildOnly, includeJoblet);
+                                    if (jobInfo.getIncludeESBFlag() >= 2) {
+                                        hasChildrenIncludeESB = true;
+                                    }
+                                }
+                            } else {
+                                Optional<JobInfo> infoOptional = jobInfos.stream().filter(info -> info.equals(jobInfo))
+                                        .findFirst();
+                                if (infoOptional.isPresent()) {
+                                    JobInfo matchJobInfo = infoOptional.get();
+                                    if (matchJobInfo.getIncludeESBFlag() >= 2) {
+                                        hasChildrenIncludeESB = true;
+                                    }
                                 }
                             }
                         }
@@ -2575,6 +2399,16 @@ public class ProcessorUtilities {
                 }
             }
         }
+
+        // checked done set the esb including option
+        if (!firstChildOnly) {
+            recordESBIncludingFlag(parentJobInfo, TalendProcessOptionConstants.ISESB_CHECKED);
+            if (hasChildrenIncludeESB) {
+                recordESBIncludingFlag(parentJobInfo, TalendProcessOptionConstants.ISESB_CHILDREN_INCLUDE);
+            }
+            esbJobs.put(esbJobKey(parentJobInfo.getJobId(), parentJobInfo.getJobVersion()), parentJobInfo.getIncludeESBFlag());
+        }
+
         return jobInfos;
     }
 
@@ -2791,9 +2625,21 @@ public class ProcessorUtilities {
     }
 
     public static boolean isEsbJob(IProcess process, boolean checkCurrentProcess) {
+        // get includeESBFlag from cache
+        if (process instanceof IProcess2) {
+            Property property = ((IProcess2) process).getProperty();
+            String esbJobKey = esbJobKey(property.getId(), property.getVersion());
+            if (esbJobs.get(esbJobKey) != null) {
+                Integer esbOptions = esbJobs.get(esbJobKey);
+                if (BitwiseOptionUtils.containOption(esbOptions, TalendProcessOptionConstants.ISESB_CHECKED)) {
+                    return checkCurrentProcess
+                            ? BitwiseOptionUtils.containOption(esbOptions, TalendProcessOptionConstants.ISESB_CURRENT_INCLUDE)
+                            : esbOptions > 2;
+                }
+            }
+        }
 
         if (process instanceof IProcess2) {
-
             if (checkCurrentProcess) {
                 for (INode n : process.getGraphicalNodes()) {
                     if (isEsbComponentName(n.getComponent().getName())) {
@@ -2819,17 +2665,6 @@ public class ProcessorUtilities {
         return false;
     }
 
-    private static void addEsbJob(JobInfo jobInfo) {
-        if (esbJobs.contains(esbJobKey(jobInfo.getJobId(), jobInfo.getJobVersion()))) {
-            return;
-        }
-        
-        esbJobs.add(esbJobKey(jobInfo.getJobId(), jobInfo.getJobVersion()));
-        if (jobInfo.getFatherJobInfo() != null) {
-            addEsbJob(jobInfo.getFatherJobInfo());
-        }
-    }
-
     private static String esbJobKey(String processId, String version) {
         return processId + "_" + version;
     }
@@ -2857,11 +2692,36 @@ public class ProcessorUtilities {
     }
     
     public static boolean isCIMode() {
-        return isCIMode;
+        // if it's CI mode , then the system property of maven.local.repository will store the value of studio
+        // m2 path,otherwise it's null
+        return System.getProperty("maven.local.repository") != null;
     }
 
-    public static void setCIMode(boolean isCIMode) {
-        ProcessorUtilities.isCIMode = isCIMode;
+
+    public static void setExportConfig(boolean export) {
+        setExportConfig(JavaUtils.JAVA_APP_NAME, null, null, export, new Date());
     }
 
+    public static boolean isJobTest(String processId, String contextName, String version) {
+        for (JobInfo jobInfo : jobList) {
+            if (jobInfo.getJobId().equals(processId)) {
+                if (contextName != null && !contextName.equals("") && !jobInfo.getContextName().equals(contextName)) {
+                    continue;
+                }
+                if (version != null && !version.equals(jobInfo.getJobVersion())) {
+                    continue;
+                }
+                return jobInfo.isTestContainer();
+            }
+        }
+        return false;
+    }
+
+    public static void setDynamicJobAndCITest(boolean dynamicJobAndCITest) {
+        isDynamicJobAndCITest = dynamicJobAndCITest;
+    }
+
+    public static boolean isDynamicJobAndCITest() {
+        return isDynamicJobAndCITest;
+    }
 }

main/plugins/org.talend.designer.maven.aether/pom.xml

@@ -11,7 +11,7 @@
   <packaging>eclipse-plugin</packaging>
   <properties>
     <maven.resolver.version>1.3.1</maven.resolver.version>
-    <wagon.version>3.0.0</wagon.version>
+    <wagon.version>3.4.3</wagon.version>
   </properties>
 
   <dependencies>

main/plugins/org.talend.designer.maven.tos/resources/build/plugins/build-helper-maven-plugin-3-0-0/pom.xml

@@ -14,6 +14,12 @@
 			<groupId>org.codehaus.mojo</groupId>
 			<artifactId>build-helper-maven-plugin</artifactId>
 			<version>3.0.0</version>
+			<exclusions>
+    <exclusion>
+     <groupId>org.apache.maven.shared</groupId>
+     <artifactId>maven-shared-utils</artifactId>
+    </exclusion>
+   </exclusions>
 		</dependency>
 	</dependencies>
 	<build>

main/plugins/org.talend.designer.maven.tos/resources/build/plugins/karaf-maven-plugin-4-2-10/pom.xml

@@ -23,6 +23,10 @@
 	              <groupId>org.apache.maven.shared</groupId>
 	              <artifactId>maven-shared-utils</artifactId>
 	            </exclusion>
+                <exclusion>
+                  <groupId>org.apache.httpcomponents</groupId>
+                  <artifactId>httpclient</artifactId>
+                </exclusion>
 			</exclusions>
 		</dependency>
 		<dependency>
@@ -91,6 +95,11 @@
 		    <artifactId>jackson-databind</artifactId>
 		    <version>2.12.0</version>
 		</dependency>
+		<dependency>
+		    <groupId>org.apache.httpcomponents</groupId>
+		    <artifactId>httpclient</artifactId>
+		    <version>4.5.13</version>
+		</dependency>
 	</dependencies>
 	<build>
 		<plugins>

main/plugins/org.talend.designer.maven.tos/resources/build/plugins/maven-bundle-plugin-5-1-1/pom.xml

@@ -75,7 +75,7 @@
 		<dependency>
 		    <groupId>com.thoughtworks.xstream</groupId>
 		    <artifactId>xstream</artifactId>
-		    <version>1.4.15</version>
+		    <version>1.4.16</version>
 		</dependency>
 	</dependencies>
 

main/plugins/org.talend.designer.maven.tos/resources/build/pom.xml

@@ -22,11 +22,23 @@
 			<groupId>org.apache.maven.plugins</groupId>
 			<artifactId>maven-clean-plugin</artifactId>
 			<version>3.0.0</version>
+			 <exclusions>
+     <exclusion>
+      <groupId>org.apache.maven.shared</groupId>
+      <artifactId>maven-shared-utils</artifactId>
+     </exclusion>
+    </exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.maven.plugins</groupId>
 			<artifactId>maven-compiler-plugin</artifactId>
 			<version>3.6.1</version>
+			<exclusions>
+    <exclusion>
+     <groupId>org.apache.maven.shared</groupId>
+     <artifactId>maven-shared-utils</artifactId>
+    </exclusion>
+   </exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.maven.plugins</groupId>
@@ -37,6 +49,12 @@
 			<groupId>org.apache.maven.plugins</groupId>
 			<artifactId>maven-jar-plugin</artifactId>
 			<version>3.2.0</version>
+			<exclusions>
+    <exclusion>
+     <groupId>org.apache.maven.shared</groupId>
+     <artifactId>maven-shared-utils</artifactId>
+    </exclusion>
+   </exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.maven.plugins</groupId>

main/plugins/org.talend.designer.maven.tos/resources/components/pom.xml

@@ -192,12 +192,12 @@
 		<dependency>
 			<groupId>org.codehaus.jackson</groupId>
 			<artifactId>jackson-core-asl</artifactId>
-			<version>1.9.14-TALEND</version>
+			<version>${jackson-codehaus.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.codehaus.jackson</groupId>
 			<artifactId>jackson-mapper-asl</artifactId>
-			<version>1.9.14-TALEND</version>
+			<version>${jackson-codehaus.version}</version>
 		</dependency>
 	</dependencies>
 	<build>

main/plugins/org.talend.designer.maven.tos/resources/pom.xml

@@ -21,5 +21,6 @@
     </modules>
     <properties>
         <m2.fasterxml.jackson.version>2.11.4</m2.fasterxml.jackson.version>
+        <jackson-codehaus.version>1.9.15-TALEND</jackson-codehaus.version>
     </properties>
 </project>

main/plugins/org.talend.designer.maven.tos/resources/surefire/pom.xml

@@ -23,7 +23,7 @@
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>
-			<version>4.13</version>
+            <version>4.13.2</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.httpcomponents</groupId>

main/plugins/org.talend.designer.maven.tos/resources/tacokit/pom.xml

@@ -10,7 +10,7 @@
 	<artifactId>studio-tacokit-dependencies</artifactId>
 	<packaging>pom</packaging>
 	<properties>
-		<tacokit.components.version>1.20.0-SNAPSHOT</tacokit.components.version>
+		<tacokit.components.version>1.21.0-SNAPSHOT</tacokit.components.version>
 	</properties>
 	<repositories>
 		<repository>

main/plugins/org.talend.designer.maven.tos/resources/tcompv1/pom.xml

@@ -11,7 +11,7 @@
     <packaging>pom</packaging>
     
     <properties>
-        <tcomp.version>1.31.0</tcomp.version>
+        <tcomp.version>1.32.0</tcomp.version>
         <slf4j.version>1.7.25</slf4j.version>
     </properties>
     

main/plugins/org.talend.designer.maven.tos/resources/unzip/pom.xml

@@ -33,7 +33,7 @@
                                 <artifactItem>
                                     <groupId>org.talend.studio</groupId>
                                     <artifactId>studio-maven-repository</artifactId>
-                                    <version>7.4.1.v20201225</version>
+                                    <version>7.4.1.v20210427</version>
                                     <type>zip</type>
                                     <overWrite>true</overWrite>
                                     <outputDirectory>${project.basedir}/../repository</outputDirectory>

main/plugins/org.talend.designer.maven/resources/templates/general/pom_codes_template.xml

@@ -17,6 +17,8 @@
 
 	<properties>
 		<talend.codes.package>@CodesPackage@</talend.codes.package>
+        <codes.compile.failOnError>true</codes.compile.failOnError>
+        <maven.compiler.failOnError>${codes.compile.failOnError}</maven.compiler.failOnError>
 	</properties>
 
 	<dependencies>

main/plugins/org.talend.designer.maven/resources/templates/general/pom_project_template.xml

@@ -31,6 +31,11 @@
 					</configuration>
 					<dependencies>
 						<!-- This dependency provides the implementation of compiler "jdt": -->
+						 <dependency>
+       <groupId>org.apache.maven.shared</groupId>
+       <artifactId>maven-shared-utils</artifactId>
+       <version>3.3.3</version>
+      </dependency>
 						<dependency>
 							<groupId>org.eclipse.tycho</groupId>
 							<artifactId>tycho-compiler-jdt</artifactId>
@@ -53,6 +58,11 @@
 							<artifactId>plexus-utils</artifactId>
 							<version>3.0.24</version>
 						</dependency>
+						<dependency>
+       <groupId>org.apache.maven.shared</groupId>
+       <artifactId>maven-shared-utils</artifactId>
+       <version>3.3.3</version>
+      </dependency>
 					</dependencies>
 				</plugin>
 				<plugin>
@@ -60,6 +70,11 @@
 					<artifactId>maven-jar-plugin</artifactId>
 					<version>3.0.2</version>
 					<dependencies>
+					 <dependency>
+       <groupId>org.apache.maven.shared</groupId>
+       <artifactId>maven-shared-utils</artifactId>
+       <version>3.3.3</version>
+      </dependency>
 						<dependency>
 							<groupId>org.codehaus.plexus</groupId>
 							<artifactId>plexus-utils</artifactId>
@@ -82,6 +97,11 @@
 					<artifactId>maven-assembly-plugin</artifactId>
 					<version>3.0.0</version>
 					<dependencies>
+					 <dependency>
+       <groupId>org.apache.maven.shared</groupId>
+       <artifactId>maven-shared-utils</artifactId>
+       <version>3.3.3</version>
+      </dependency>
 						<dependency>
 							<groupId>org.codehaus.plexus</groupId>
 							<artifactId>plexus-utils</artifactId>
@@ -108,7 +128,12 @@
 							<groupId>org.codehaus.plexus</groupId>
 							<artifactId>plexus-utils</artifactId>
 							<version>3.0.24</version>
-						</dependency>
+                        </dependency>
+                        <dependency>
+                            <groupId>junit</groupId>
+                            <artifactId>junit</artifactId>
+                            <version>4.13.2</version>
+                        </dependency>
 					</dependencies>
 				</plugin>
 				<plugin>

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/launch/MavenCommandLauncher.java

@@ -97,7 +97,7 @@ public abstract class MavenCommandLauncher {
 
     private Map<String, Object> argumentsMap;
 
-    private static final String REGEX_TEST_CASE_FAILURES_STR = "\\[ERROR\\] Tests run:.*\\[ERROR\\] There are test\\p{Print}+\n";
+    private static final String REGEX_TEST_CASE_FAILURES_STR = "\\[ERROR\\] There are test failures.*\n";
 
     private static final Pattern REGEX_TEST_CASE_FAILURES = Pattern.compile(REGEX_TEST_CASE_FAILURES_STR, Pattern.DOTALL);
 

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/AggregatorPomsHelper.java

@@ -244,11 +244,6 @@ public class AggregatorPomsHelper {
         createTemplatePom.create(monitor);
     }
 
-    public static void buildAndInstallCodesProject(IProgressMonitor monitor, ERepositoryObjectType codeType)
-            throws Exception {
-        buildAndInstallCodesProject(monitor, codeType, true, false);
-    }
-
     public static void buildAndInstallCodesProject(IProgressMonitor monitor, ERepositoryObjectType codeType,
             boolean install, boolean forceBuild) throws Exception {
         if (forceBuild || !BuildCacheManager.getInstance().isCodesBuild(codeType)) {
@@ -271,33 +266,29 @@ public class AggregatorPomsHelper {
         }
     }
 
-    // only compile for global/custom code projects
     public static void buildCodesProject() {
-        IRunProcessService service = IRunProcessService.get();
-        if (service == null) {
-            return;
-        }
         IProgressMonitor monitor = new NullProgressMonitor();
         ERepositoryObjectType.getAllTypesOfCodes().forEach(type -> {
             try {
-                buildAndInstallCodesProject(monitor, type, false, false);
+                buildAndInstallCodesProject(monitor, type, true, false);
             } catch (Exception e) {
                 ExceptionHandler.process(e);
             }
         });
-        Set<CodesJarInfo> jarsToUpdate = CodesJarResourceCache.getAllCodesJars().stream()
-                .filter(info -> CodesJarM2CacheManager.needUpdateCodesJarProject(info)).collect(Collectors.toSet());
-        jarsToUpdate.stream().map(info -> service.getTalendCodesJarJavaProject(info)).forEach(p -> {
-            try {
-                p.buildModules(monitor, null, null);
-            } catch (Exception e) {
-                ExceptionHandler.process(e);
-            }
-        });
-        String currentProjectName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
-        jarsToUpdate.stream().filter(info -> !currentProjectName.equals(info.getProjectTechName()))
-                .forEach(info -> service.deleteTalendCodesJarProject(info, false));
+        CodesJarM2CacheManager.updateCodesJarProject(monitor);
+    }
 
+    public static void buildCodesProject(IProgressMonitor monitor, Set<CodesJarInfo> toUpdate) {
+        ERepositoryObjectType.getAllTypesOfCodes().forEach(type -> {
+            try {
+                buildAndInstallCodesProject(monitor, type, true, false);
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+        });
+        CodesJarM2CacheManager.updateCodesJarProject(monitor, toUpdate.stream()
+                .filter(info -> CodesJarM2CacheManager.needUpdateCodesJarProject(info)).collect(Collectors.toSet()), false, false,
+                false);
     }
 
     public void updateRefProjectModules(List<ProjectReference> references, IProgressMonitor monitor) {

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/tools/CodesJarM2CacheManager.java

@@ -68,7 +68,6 @@ import org.talend.designer.maven.tools.creator.CreateMavenRoutinesJarPom;
 import org.talend.designer.maven.utils.PomIdsHelper;
 import org.talend.designer.maven.utils.PomUtil;
 import org.talend.designer.runprocess.IRunProcessService;
-import org.talend.repository.ProjectManager;
 import org.talend.repository.RepositoryWorkUnit;
 import org.talend.utils.io.FilesUtils;
 
@@ -259,13 +258,23 @@ public class CodesJarM2CacheManager {
         updateCodesJarProject(monitor, false, false, false);
     }
 
+    public static void updateCodesJarProjectForLogon(IProgressMonitor monitor) {
+        Set<CodesJarInfo> allCodesJars = CodesJarResourceCache.getAllCodesJars();
+        Set<CodesJarInfo> toUpdate = allCodesJars.stream()
+                .filter(info -> info.isInCurrentMainProject() && needUpdateCodesJarProject(info)).collect(Collectors.toSet());
+        allCodesJars.removeAll(toUpdate);
+        // compile directly for the rest of jar projects
+        allCodesJars.stream().map(info -> IRunProcessService.get().getExistingTalendCodesJarProject(info)).filter(p -> p != null)
+                .forEach(p -> p.buildWholeCodeProject());
+        updateCodesJarProject(monitor, toUpdate, false, false, false);
+    }
+
     public static void updateCodesJarProject(IProgressMonitor monitor, boolean forceBuild, boolean onlyCurrentProject,
             boolean syncCode) {
         Set<CodesJarInfo> toUpdate;
         if (onlyCurrentProject) {
-            String currentProject = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
-            toUpdate = CodesJarResourceCache.getAllCodesJars().stream().filter(
-                    info -> info.getProjectTechName().equals(currentProject) && (forceBuild || needUpdateCodesJarProject(info)))
+            toUpdate = CodesJarResourceCache.getAllCodesJars().stream()
+                    .filter(info -> info.isInCurrentMainProject() && (forceBuild || needUpdateCodesJarProject(info)))
                     .collect(Collectors.toSet());
         } else {
             toUpdate = CodesJarResourceCache.getAllCodesJars().stream()

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/ClasspathsJarGenerator.java

@@ -49,6 +49,11 @@ public class ClasspathsJarGenerator {
 
     public static String createJar(Property property, String classpath, String separator, boolean isRelativePath)
             throws Exception {
+        return createJar(property, classpath, separator, isRelativePath, false);
+    }
+
+    public static String createJar(Property property, String classpath, String separator, boolean isRelativePath,
+            boolean usingTempDir) throws Exception {
         String newClasspath = generateClasspathForManifest(classpath, separator, isRelativePath);
 
         Manifest manifest = new Manifest();
@@ -57,7 +62,7 @@ public class ClasspathsJarGenerator {
         a.put(Attributes.Name.IMPLEMENTATION_VENDOR, "Talend Open Studio"); //$NON-NLS-1$
         a.put(Attributes.Name.CLASS_PATH, newClasspath);
 
-        String jarLocation = getJarLocation(property);
+        String jarLocation = getJarLocation(property, usingTempDir);
         File jarFile = new File(jarLocation);
         if (!jarFile.exists()) {
             jarFile.createNewFile();
@@ -122,8 +127,8 @@ public class ClasspathsJarGenerator {
         return finalClasspath.toString().trim();
     }
 
-    public static String getClasspathFromManifest(Property property) throws Exception {
-        String jarLocation = getJarLocation(property);
+    public static String getClasspathFromManifest(Property property, boolean usingTempDir) throws Exception {
+        String jarLocation = getJarLocation(property, usingTempDir);
         JarInputStream stream = null;
         try {
             stream = new JarInputStream(new FileInputStream(jarLocation));
@@ -148,10 +153,13 @@ public class ClasspathsJarGenerator {
         return str;
     }
 
-    private static String getJarLocation(Property property) {
+    private static String getJarLocation(Property property, boolean usingTempDir) {
         ITalendProcessJavaProject jobProject = getRunProcessService().getTalendJobJavaProject(property);
-        String jarLocation = jobProject.getTargetFolder().getFile(CLASSPATHS_JAR_NAME).getLocation().toPortableString();
-        return jarLocation;
+        if (usingTempDir) {
+            return jobProject.getTempFolder().getFile(CLASSPATHS_JAR_NAME).getLocation().toPortableString();
+        } else {
+            return jobProject.getTargetFolder().getFile(CLASSPATHS_JAR_NAME).getLocation().toPortableString();
+        }
     }
 
     private static IRunProcessService getRunProcessService() {

main/plugins/org.talend.designer.maven/src/main/java/org/talend/designer/maven/utils/PomIdsHelper.java

@@ -355,8 +355,9 @@ public class PomIdsHelper {
 
     public static boolean skipFolders() {
         String projectTechName = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
-        ProjectPreferenceManager manager = getPreferenceManager(projectTechName);
-        return manager.getBoolean(MavenConstants.SKIP_FOLDERS);
+        Project project = ProjectManager.getInstance().getProjectFromProjectTechLabel(projectTechName);
+        ProjectPreferenceManager preferenceManager = new ProjectPreferenceManager(project, DesignerMavenPlugin.PLUGIN_ID, false);
+        return preferenceManager.getBoolean(MavenConstants.SKIP_FOLDERS);
     }
 
     public static boolean isValidGroupId(String text) {

main/plugins/org.talend.libraries.apache.http/.classpath

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
 	<classpathentry exported="true" kind="lib" path="lib/fluent-hc-4.2.6.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/httpclient-4.5.3.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/httpclient-4.5.13.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/httpcore-4.4.6.jar"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>

main/plugins/org.talend.libraries.apache.http/META-INF/MANIFEST.MF

@@ -6,7 +6,7 @@ Bundle-Version: 7.4.1.qualifier
 Bundle-ActivationPolicy: lazy
 Eclipse-BundleShape: dir
 Bundle-ClassPath: ./,
- lib/httpclient-4.5.3.jar,
+ lib/httpclient-4.5.13.jar,
  lib/httpcore-4.4.6.jar,
  lib/fluent-hc-4.2.6.jar
 Export-Package: org.apache.http,

main/plugins/org.talend.libraries.apache.http/pom.xml

@@ -33,7 +33,7 @@
 								<artifactItem>
 									<groupId>org.apache.httpcomponents</groupId>
   									<artifactId>httpclient</artifactId>
-  									<version>4.5.3</version>
+  									<version>4.5.13</version>
 								</artifactItem>
 								<artifactItem>
 									<groupId>org.apache.httpcomponents</groupId>

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/ui/startup/ShareLibsSynchronizer.java

@@ -12,14 +12,10 @@
 // ============================================================================
 package org.talend.librariesmanager.ui.startup;
 
-import org.eclipse.ui.IStartup;
-
 import java.util.logging.Logger;
-import org.eclipse.core.runtime.preferences.IEclipsePreferences;
-import org.eclipse.core.runtime.preferences.InstanceScope;
-import org.talend.commons.exception.ExceptionHandler;
-import org.talend.commons.utils.network.NetworkUtil;
-import org.talend.core.prefs.ITalendCorePrefConstants;
+
+import org.eclipse.ui.IStartup;
+import org.talend.librariesmanager.prefs.LibrariesManagerUtils;
 
 /**
  * created by wchen on 2015-6-15 Detailled comment
@@ -41,21 +37,12 @@ public class ShareLibsSynchronizer implements IStartup {
         ShareCILibsJob ciJob = new ShareCILibsJob();
         ciJob.schedule();
 
-        if (shareLibsAtStartup()) {
+        if (LibrariesManagerUtils.shareLibsAtStartup()) {
             ShareLibsJob job = new ShareLibsJob();
             job.schedule();
+        } else {
+            LOGGER.info("Skip sharing libraries");
         }
     }
 
-    public boolean shareLibsAtStartup() {
-        boolean ret = ITalendCorePrefConstants.NEXUS_SHARE_LIBS_DEFAULT;
-        try {
-            IEclipsePreferences node = InstanceScope.INSTANCE.getNode(NetworkUtil.ORG_TALEND_DESIGNER_CORE);
-            ret = node.getBoolean(ITalendCorePrefConstants.NEXUS_SHARE_LIBS, ITalendCorePrefConstants.NEXUS_SHARE_LIBS_DEFAULT);
-        } catch (Throwable e) {
-            ExceptionHandler.process(e);
-        }
-        LOGGER.info("shareLibsAtStartup: " + ret);
-        return ret;
-    }
 }

main/plugins/org.talend.librariesmanager.ui/src/main/java/org/talend/librariesmanager/utils/RemoteModulesHelper.java

@@ -115,7 +115,7 @@ public class RemoteModulesHelper {
         @Override
         public void run(IProgressMonitor monitor) throws InvocationTargetException, InterruptedException {
             final Set<String> mavenUrisTofetch = new HashSet<String>(contextMap.keySet());
-            if (LibraryDataService.getInstance().isBuildLibrariesData()) {
+            if (LibraryDataService.isBuildLibrariesData()) {
                 LibraryDataService.getInstance().buildLibraryLicenseData(mavenUrisTofetch);
             }
             monitor.beginTask(Messages.getString("RemoteModulesHelper.fetch.module.info"), mavenUrisTofetch.size() * 10 + 10);//$NON-NLS-1$
@@ -160,7 +160,7 @@ public class RemoteModulesHelper {
                 if (useLocalLicenseData) {
                     searchFromLocalDataFile(mavenUrisTofetch, monitor);
                     addCachedModulesToToBeInstallModules(toInstall, mavenUrisTofetch, contextMap, remoteCache);
-                } 
+                }
                 if (!onlyUseLocalLicenseData || LibraryDataService.getInstance().isBuildLibrariesData()) {
                     searchFromRemoteNexus(mavenUrisTofetch, monitor);
                     addCachedModulesToToBeInstallModules(toInstall, mavenUrisTofetch, contextMap, remoteCache);  
@@ -270,6 +270,13 @@ public class RemoteModulesHelper {
                 String uriToCheck = iterator.next();
                 final MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(uriToCheck);
                 if (parseMvnUrl != null) {
+                    String repositoryUrl = parseMvnUrl.getRepositoryUrl();
+                    if (!LibraryDataService.getInstance().isBuildLibrariesData()) {
+                        if (StringUtils.isNotEmpty(repositoryUrl)
+                                && parseMvnUrl.getGroupId().startsWith(MavenConstants.APACHE_GROUP_ID)) {
+                            continue;
+                        }
+                    }
                     service.fillLibraryDataByRemote(uriToCheck, parseMvnUrl);
                     if (!MavenConstants.DOWNLOAD_MANUAL.equals(parseMvnUrl.getDistribution())) {
                         artifactList.add(parseMvnUrl);
@@ -370,7 +377,7 @@ public class RemoteModulesHelper {
                         }
                     }
 
-                    toInstall.add(moduleToInstall);
+                    toInstall.add(moduleToInstall.clone());
                     iterator.remove();
                 }
             }
@@ -641,7 +648,7 @@ public class RemoteModulesHelper {
     public RemoteModulesFetchRunnable getNotInstalledModulesRunnable(List<ModuleNeeded> neededModules,
             List<ModuleToInstall> toInstall, boolean collectModulesWithJarName, boolean useLocalLicenseData, boolean onlyUseLocalLicenseData) {
         Map<String, List<ModuleNeeded>> contextMap = new HashMap<String, List<ModuleNeeded>>();
-        ILibraryManagerService librairesManagerService = (ILibraryManagerService) GlobalServiceRegister.getDefault()
+        ILibraryManagerService librairesManagerService = GlobalServiceRegister.getDefault()
                 .getService(ILibraryManagerService.class);
         // collect mvnuri and modules incase many modules have the same mvnuri
         final Iterator<ModuleNeeded> iterator = neededModules.iterator();

main/plugins/org.talend.librariesmanager/pom.xml

@@ -48,7 +48,7 @@
 								<artifactItem>
 									<groupId>org.slf4j</groupId>
 									<artifactId>slf4j-api</artifactId>
-									<version>1.7.25</version>
+									<version>1.7.25</version> <!-- Please sync the version with /org.talend.designer.maven.lite/resources/required/pom.xml -->
 									<overWrite>true</overWrite>
 								</artifactItem>
 							</artifactItems>

main/plugins/org.talend.librariesmanager/src/main/java/messages.properties

@@ -28,6 +28,8 @@ ShareLibsJob.message=Share libraries from maven to {0} server
 ShareLibsJob.sharingLibraries=Sharing libraries : {0}
 ShareLibsJob.monitor.cancelled=User cancelled.
 
+LocalLibraryManager.shareLibsForCustomponents=Sharing libraries for custom components
+
 ExtensionModuleManager.moduleGroup.cycleReference=Module group {0} was cycle referenced!
 NexusRepository.checkConnection.successMsg=Connection successful. 
 NexusRepository.checkConnection.invalidParam=Invalid Nexus Parameter. 

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/librarydata/LibraryDataService.java

@@ -67,11 +67,17 @@ public class LibraryDataService {
 
     private static boolean buildLibraryIfFileMissing = true;
 
-    private boolean buildLibraryLicense = false;
+    /**
+     * make it as static to avoid call of getInstance, so that we can avoid dead lock in some cases
+     */
+    private static boolean buildLibraryLicense = Boolean
+            .valueOf(System.getProperty(KEY_LIBRARIES_BUILD_LICENSE, Boolean.FALSE.toString()));
 
-    private boolean buildLibraryIfLicenseMissing = false;
+    private static boolean buildLibraryIfLicenseMissing = Boolean
+            .valueOf(System.getProperty(KEY_BUILD_LIBRARY_IF_LICENSE_MISSING, Boolean.FALSE.toString()));
 
-    private boolean buildLibraryJarFile = false;
+    private static boolean buildLibraryJarFile = Boolean
+            .valueOf(System.getProperty(KEY_LIBRARIES_BUILD_JAR, Boolean.FALSE.toString()));
 
     private static final Map<String, Library> mvnToLibraryMap = new ConcurrentHashMap<String, Library>();
 
@@ -86,10 +92,6 @@ public class LibraryDataService {
     private LibraryLicense unknownLicense;;
 
     private LibraryDataService() {
-        buildLibraryLicense = Boolean.valueOf(System.getProperty(KEY_LIBRARIES_BUILD_LICENSE, Boolean.FALSE.toString()));
-        buildLibraryIfLicenseMissing = Boolean
-                .valueOf(System.getProperty(KEY_BUILD_LIBRARY_IF_LICENSE_MISSING, Boolean.FALSE.toString()));
-        buildLibraryJarFile = Boolean.valueOf(System.getProperty(KEY_LIBRARIES_BUILD_JAR, Boolean.FALSE.toString()));
         File studioLibraryDataFile = getStudioLibraryDataFile();
         if (buildLibraryLicense) {
             if (studioLibraryDataFile.exists()) {
@@ -329,7 +331,11 @@ public class LibraryDataService {
         return value.toString();
     }
 
-    public boolean isBuildLibrariesData() {
+    public static boolean isBuildLibrariesData() {
+        /**
+         * make the method as static to avoid call of getInstance, so that we can avoid dead lock in some cases
+         */
+
         if (buildLibraryLicense) {
             return true;
         }
@@ -348,7 +354,7 @@ public class LibraryDataService {
         return new File(Platform.getConfigurationLocation().getURL().getPath(), LIBRARIES_DATA_FILE_NAME);
     }
 
-    private File getStudioLibraryDataFile() {
+    private static File getStudioLibraryDataFile() {
         String folder = System.getProperty(KEY_LIBRARIES_DATA_FOLDER);
         if (folder == null) {
             folder = new File(Platform.getInstallLocation().getURL().getPath(), "configuration").getAbsolutePath(); //$NON-NLS-1$

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/maven/ShareLibrareisHelper.java

@@ -13,7 +13,6 @@
 package org.talend.librariesmanager.maven;
 
 import java.io.File;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -77,13 +76,13 @@ public abstract class ShareLibrareisHelper {
             // collect groupId to search
             Set<String> snapshotGroupIdSet = new HashSet<String>();
             Set<String> releaseGroupIdSet = new HashSet<String>();
-            checkCancel(monitor);
+            ShareLibrariesUtil.checkCancel(monitor);
             for (ModuleNeeded module : filesToShare.keySet()) {
-                checkCancel(monitor);
+                ShareLibrariesUtil.checkCancel(monitor);
                 if (module.getMavenUri() != null) {
                     MavenArtifact parseMvnUrl = MavenUrlHelper.parseMvnUrl(module.getMavenUri());
                     if (parseMvnUrl != null) {
-                        if (isSnapshotVersion(parseMvnUrl.getVersion())) {
+                        if (ShareLibrariesUtil.isSnapshotVersion(parseMvnUrl.getVersion())) {
                             snapshotGroupIdSet.add(parseMvnUrl.getGroupId());
                         } else {
                             releaseGroupIdSet.add(parseMvnUrl.getGroupId());
@@ -93,22 +92,24 @@ public abstract class ShareLibrareisHelper {
             }
 
             // search from custom artifact repositories if any
-            seachArtifacts(monitor, customerRepHandler, snapshotArtifactMap, releaseArtifactMap, snapshotGroupIdSet,
+            ShareLibrariesUtil.seachArtifacts(monitor, customerRepHandler, snapshotArtifactMap, releaseArtifactMap,
+                    snapshotGroupIdSet,
                     releaseGroupIdSet);
 
             try {
                 // search from proxy artifact repository if any
-                seachArtifacts(monitor, proxyArtifactHandler, snapshotArtifactMap, releaseArtifactMap, snapshotGroupIdSet,
+                ShareLibrariesUtil.seachArtifacts(monitor, proxyArtifactHandler, snapshotArtifactMap, releaseArtifactMap,
+                        snapshotGroupIdSet,
                         releaseGroupIdSet);
             } catch (Exception e) {
                 ExceptionHandler.process(e);
             }
-            checkCancel(monitor);
+            ShareLibrariesUtil.checkCancel(monitor);
 
             Iterator<ModuleNeeded> iterator = filesToShare.keySet().iterator();
             Map<File, MavenArtifact> shareFiles = new HashMap<>();
             while (iterator.hasNext()) {
-                checkCancel(monitor);
+                ShareLibrariesUtil.checkCancel(monitor);
                 ModuleNeeded next = iterator.next();
                 File file = filesToShare.get(next);
                 MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(next.getMavenUri());
@@ -118,7 +119,7 @@ public abstract class ShareLibrareisHelper {
                 // If from custom component definition file
                 if (LocalLibraryManager.isSystemCacheFile(file.getName())
                         || (LocalLibraryManager.isComponentDefinitionFileType(file.getName())
-                                && isTalendLibraryGroupId(artifact))) {
+                                && ShareLibrariesUtil.isTalendLibraryGroupId(artifact))) {
                     continue;
                 }
                 try {
@@ -128,7 +129,7 @@ public abstract class ShareLibrareisHelper {
                 } catch (NumberFormatException e) {
                     //
                 }
-                boolean isSnapshotVersion = isSnapshotVersion(artifact.getVersion());
+                boolean isSnapshotVersion = ShareLibrariesUtil.isSnapshotVersion(artifact.getVersion());
                 String key = ShareLibrariesUtil.getArtifactKey(artifact, isSnapshotVersion);
                 List<MavenArtifact> artifactList = null;
                 if (isSnapshotVersion) {
@@ -151,7 +152,7 @@ public abstract class ShareLibrareisHelper {
             }
             SubMonitor mainSubMonitor = SubMonitor.convert(monitor, shareFiles.size());
             for (Map.Entry<File, MavenArtifact> entry : shareFiles.entrySet()) {
-                checkCancel(monitor);
+                ShareLibrariesUtil.checkCancel(monitor);
                 try {
                     File k = entry.getKey();
                     MavenArtifact v = entry.getValue();
@@ -173,63 +174,12 @@ public abstract class ShareLibrareisHelper {
 
     }
 
-    /**
-     * Search artifacts based on given snapshotGroupIdSet and releaseGroupIdSet from remote artifact repositories
-     * represented by artifactHandler, the search results are put to snapshotArtifactMap and releaseArtifactMap
-     */
-    protected void seachArtifacts(IProgressMonitor monitor, IRepositoryArtifactHandler artifactHandler,
-            Map<String, List<MavenArtifact>> snapshotArtifactMap, Map<String, List<MavenArtifact>> releaseArtifactMap,
-            Set<String> snapshotGroupIdSet, Set<String> releaseGroupIdSet) throws Exception {
-        if (artifactHandler != null) {
-            checkCancel(monitor);
-            List<MavenArtifact> searchResults = new ArrayList<MavenArtifact>();
-            for (String groupId : releaseGroupIdSet) {
-                searchResults = artifactHandler.search(groupId, null, null, true, false);
-                if (searchResults != null) {
-                    for (MavenArtifact result : searchResults) {
-                        checkCancel(monitor);
-                        ShareLibrariesUtil.putArtifactToMap(result, releaseArtifactMap, false);
-                    }
-                }
-            }
-            checkCancel(monitor);
-            for (String groupId : snapshotGroupIdSet) {
-                searchResults = artifactHandler.search(groupId, null, null, false, true);
-                if (searchResults != null) {
-                    for (MavenArtifact result : searchResults) {
-                        checkCancel(monitor);
-                        ShareLibrariesUtil.putArtifactToMap(result, snapshotArtifactMap, true);
-                    }
-                }
-            }
-        }
-    }
-
-    private boolean isTalendLibraryGroupId(MavenArtifact artifact) {
-        if ("org.talend.libraries".equalsIgnoreCase(artifact.getGroupId())) {
-            return true;
-        }
-        return false;
-    }
-
-    private boolean isSnapshotVersion(String version) {
-        if (version != null && version.toUpperCase().endsWith(MavenUrlHelper.VERSION_SNAPSHOT)) {
-            return true;
-        }
-        return false;
-    }
     private void setJobName(Job job, String jobName) {
         if (job != null) {
             job.setName(jobName);
         }
     }
 
-    protected void checkCancel(IProgressMonitor monitor) throws InterruptedException {
-        if (monitor.isCanceled()) {
-            throw new InterruptedException(Messages.getString("ShareLibsJob.monitor.cancelled"));
-        }
-    }
-
     public abstract Map<ModuleNeeded, File> getFilesToShare(IProgressMonitor monitor);
 
     public abstract void shareToRepository(File jarFile, MavenArtifact module) throws Exception;

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/ModulesNeededProvider.java

@@ -83,6 +83,7 @@ import org.talend.core.model.routines.IRoutinesProvider;
 import org.talend.core.runtime.maven.MavenArtifact;
 import org.talend.core.runtime.maven.MavenUrlHelper;
 import org.talend.core.runtime.process.TalendProcessOptionConstants;
+import org.talend.core.ui.ITestContainerProviderService;
 import org.talend.core.utils.CodesJarResourceCache;
 import org.talend.core.utils.TalendCacheUtils;
 import org.talend.core.utils.TalendQuoteUtils;
@@ -575,6 +576,14 @@ public class ModulesNeededProvider {
         }
         EList<RoutinesParameterType> routinesParameterTypes = null;
         if (item instanceof ProcessItem) {
+            ITestContainerProviderService testcaseService = ITestContainerProviderService.get();
+            if (testcaseService != null && testcaseService.isTestContainerItem(item)) {
+                try {
+                    item = testcaseService.getParentJobItem(item);
+                } catch (PersistenceException e) {
+                    ExceptionHandler.process(e);
+                }
+            }
             if (((ProcessItem) item).getProcess() != null && ((ProcessItem) item).getProcess().getParameters() != null) {
                 routinesParameterTypes = ((ProcessItem) item).getProcess().getParameters().getRoutinesParameter();
             }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/LocalLibraryManager.java

@@ -29,7 +29,6 @@ import java.util.Collection;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -68,10 +67,8 @@ import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
 import org.talend.core.model.general.ModuleStatusProvider;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.nexus.ArtifactRepositoryBean;
-import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.nexus.NexusConstants;
 import org.talend.core.nexus.NexusServerUtils;
-import org.talend.core.nexus.RepositoryArtifactHandlerManager;
 import org.talend.core.nexus.TalendLibsServerManager;
 import org.talend.core.nexus.TalendMavenResolver;
 import org.talend.core.prefs.ITalendCorePrefConstants;
@@ -83,10 +80,10 @@ import org.talend.core.utils.TalendQuoteUtils;
 import org.talend.designer.maven.tools.BuildCacheManager;
 import org.talend.designer.maven.utils.PomUtil;
 import org.talend.designer.runprocess.IRunProcessService;
+import org.talend.librariesmanager.i18n.Messages;
 import org.talend.librariesmanager.maven.MavenArtifactsHandler;
 import org.talend.librariesmanager.model.ExtensionModuleManager;
 import org.talend.librariesmanager.model.ModulesNeededProvider;
-import org.talend.librariesmanager.nexus.utils.ShareLibrariesUtil;
 import org.talend.librariesmanager.prefs.LibrariesManagerUtils;
 import org.talend.osgi.hook.notification.JarMissingObservable;
 
@@ -124,9 +121,13 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
             add(".javajet");
             add(".png");
             add(".jpg");
-            add("_java.xml");
+            add(".xml");
             add(".properties");
             add(".txt");
+            add(".class");
+            add(".mf");
+            add(".rsa");
+            add(".sf");
         }
     };
 
@@ -1287,7 +1288,7 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
                 warnDuplicated(modules, duplicateMavenUri, "Maven Uri:");
             }
         }
-
+        
         if (service != null) {
             calculateModulesIndexFromComponentFolder(service, platformURLMap);
         }
@@ -1348,138 +1349,87 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         }
     }
     
-	private boolean isExtComponentProvider(String id) {
-		if ("org.talend.designer.components.model.UserComponentsProvider".equals(id)
-				|| "org.talend.designer.codegen.components.model.SharedStudioUserComponentProvider".equals(id)
-				|| "org.talend.designer.components.exchange.ExchangeComponentsProvider".equals(id)
-				|| "org.talend.designer.components.exchange.SharedStudioExchangeComponentsProvider".equals(id)) {
-			return true;
-		}
-		return false;
-	}
-
-	private void deployLibsFromCustomComponents(IComponentsService service, Map<String, String> platformURLMap) {
-		Set<File> needToDeploy = new HashSet<>();
-		List<ComponentProviderInfo> componentsFolders = service.getComponentsFactory().getComponentsProvidersInfo();
-		for (ComponentProviderInfo providerInfo : componentsFolders) {
-			String id = providerInfo.getId();
-			try {
-				File file = new File(providerInfo.getLocation());
-				if (isExtComponentProvider(id)) {
-					if (file.isDirectory()) {
-						List<File> jarFiles = FilesUtils.getJarFilesFromFolder(file, null);
-						if (jarFiles.size() > 0) {
-							for (File jarFile : jarFiles) {
-								String name = jarFile.getName();
-								if (!canDeployFromCustomComponentFolder(name) || platformURLMap.get(name) != null) {
-									continue;
-								}
-								needToDeploy.add(jarFile);
-							}
-						}
-					} else {
-						if (platformURLMap.get(file.getName()) != null) {
-							continue;
-						}
-						needToDeploy.add(file);
-					}
-				}
-			} catch (Exception e) {
-				ExceptionHandler.process(e);
-				continue;
-			}
-		}
-
-        // deploy needed jars for User and Exchange component providers
-        Map<String, List<MavenArtifact>> snapshotArtifactMap = new HashMap<String, List<MavenArtifact>>();
-        Map<String, List<MavenArtifact>> releaseArtifactMap = new HashMap<String, List<MavenArtifact>>();
-        if (!needToDeploy.isEmpty()) {
-            // search on nexus to avoid deploy the jar many times
-            Set<File> existFiles = new HashSet<>();
-            ArtifactRepositoryBean customNexusServer = TalendLibsServerManager.getInstance().getCustomNexusServer();
-            IRepositoryArtifactHandler customerRepHandler = RepositoryArtifactHandlerManager
-                    .getRepositoryHandler(customNexusServer);
-            if (customerRepHandler != null) {
-                List<MavenArtifact> snapshotResult = new ArrayList<>();
-                List<MavenArtifact> releaseResult = new ArrayList<>();
-                try {
-                    snapshotResult = customerRepHandler.search(MavenConstants.DEFAULT_LIB_GROUP_ID, null, null, false, true);
-                    if (snapshotResult != null) {
-                        for (MavenArtifact result : snapshotResult) {
-                            ShareLibrariesUtil.putArtifactToMap(result, snapshotArtifactMap, true);
-                        }
-                    }
-                    releaseResult = customerRepHandler.search(MavenConstants.DEFAULT_LIB_GROUP_ID, null, null, true, false);
-                    if (releaseResult != null) {
-                        for (MavenArtifact result : releaseResult) {
-                            ShareLibrariesUtil.putArtifactToMap(result, releaseArtifactMap, false);
-                        }
-                    }
-                } catch (Exception e) {
-                    ExceptionHandler.process(e);
-                }
-            }
-            for(File exsitFile:needToDeploy) {
-                if (customerRepHandler != null) {
-                  try {
-                        String name = exsitFile.getName();
-                        String mvnUrlSnapshot = MavenUrlHelper.generateMvnUrlForJarName(name, true, true);
-                        MavenArtifact artifactSnapshot = MavenUrlHelper.parseMvnUrl(mvnUrlSnapshot);
-                        String keySnapshot = ShareLibrariesUtil.getArtifactKey(artifactSnapshot, true);
-                        List<MavenArtifact> artifactListSnapshot = null;
-                        artifactListSnapshot = snapshotArtifactMap.get(keySnapshot);
-                        // snapshot
-                        if (artifactListSnapshot != null && artifactListSnapshot.size() > 0) {
-                            if (ShareLibrariesUtil.isSameFileWithRemote(exsitFile, artifactListSnapshot, customNexusServer,
-                                    customerRepHandler, false)) {
-                                existFiles.add(exsitFile);
-                            }
-                        }
-                        // release
-                        String mvnUrlRelease = MavenUrlHelper.generateMvnUrlForJarName(name, true, false);
-                        MavenArtifact artifactRelease = MavenUrlHelper.parseMvnUrl(mvnUrlRelease);
-                        String keyRelease = ShareLibrariesUtil.getArtifactKey(artifactRelease, false);
-                        List<MavenArtifact> artifactListRelease = null;
-                        artifactListRelease = releaseArtifactMap.get(keyRelease);
-                        if (artifactListRelease != null && artifactListRelease.size() > 0) {
-                            if (ShareLibrariesUtil.isSameFileWithRemote(exsitFile, artifactListRelease, customNexusServer,
-                                    customerRepHandler, false)) {
-                                existFiles.add(exsitFile);
-                            }
-                        }
-                  }catch(Exception e) {
-                      ExceptionHandler.process(e);
-                  }
-                }
-            }
-            needToDeploy.removeAll(existFiles);
-            // check sha code to avoid same jar in diff component depoly multi times
-            Map<String, File> shaMap = new HashMap<>();
-            Iterator<File> it = needToDeploy.iterator();
-            while (it.hasNext()) {
-                try {
-                    File dupFile = it.next();
-                    String localFileShaCode = DigestUtils.shaHex(new FileInputStream(dupFile));
-                    if (shaMap.get(localFileShaCode) == null) {
-                        shaMap.put(localFileShaCode, dupFile);
-                    } else {
-                        it.remove();
-                    }
-                } catch (Exception e) {
-                    ExceptionHandler.process(e);
-                }
-            }
-            for (File file : needToDeploy) {
-                try {
-                    // deploy as release version if can't find mvn url from index
-                    install(file, null, true, true);
-                } catch (Exception e) {
-                    ExceptionHandler.process(e);
-                    continue;
-                }
-            }
-
+   private boolean isExtComponentProvider(String id) {
+       if ("org.talend.designer.components.model.UserComponentsProvider".equals(id)
+               || "org.talend.designer.codegen.components.model.SharedStudioUserComponentProvider".equals(id)
+               || "org.talend.designer.components.exchange.ExchangeComponentsProvider".equals(id)
+               || "org.talend.designer.components.exchange.SharedStudioExchangeComponentsProvider".equals(id)) {
+           return true;
+       }
+       return false;
+   }
+
+    private void deployLibsFromCustomComponents(IComponentsService service, Map<String, String> platformURLMap) {
+        boolean deployToRemote = true;
+        if (!LibrariesManagerUtils.shareLibsAtStartup()) {
+            log.info("Skip deploying libs from custom components");
+            deployToRemote = false;
         }
+
+        Map<File, Set<MavenArtifact>> needToDeploy = new HashMap<File, Set<MavenArtifact>>();
+        List<ComponentProviderInfo> componentsFolders = service.getComponentsFactory().getComponentsProvidersInfo();
+        for (ComponentProviderInfo providerInfo : componentsFolders) {
+            String id = providerInfo.getId();
+            try {
+                File file = new File(providerInfo.getLocation());
+                if (isExtComponentProvider(id)) {
+                    if (file.isDirectory()) {
+                        List<File> jarFiles = FilesUtils.getJarFilesFromFolder(file, null);
+                        if (jarFiles.size() > 0) {
+                            for (File jarFile : jarFiles) {
+                                String name = jarFile.getName();
+                                if (!canDeployFromCustomComponentFolder(name) || platformURLMap.get(name) != null) {
+                                    continue;
+                                }
+
+                                collectLibModules(jarFile, needToDeploy);
+                            }
+                        }
+                    } else {
+                        if (!canDeployFromCustomComponentFolder(file.getName()) || platformURLMap.get(file.getName()) != null) {
+                            continue;
+                        }
+                        collectLibModules(file, needToDeploy);
+                    }
+                }
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+                continue;
+            }
+        }
+
+        // first install them locally
+        needToDeploy.forEach((k, v) -> {
+            try {
+                // install as release version if can't find mvn url from index
+                install(k, null, false, true);
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+        });
+
+        if (!deployToRemote) {
+            return;
+        }
+
+        ShareComponentsLibsJob shareJob = new ShareComponentsLibsJob(
+                Messages.getString("LocalLibraryManager.shareLibsForCustomponents"), needToDeploy, deployer);
+        shareJob.schedule();
+    }
+
+    private void collectLibModules(File jarFile, Map<File, Set<MavenArtifact>> needToDeploy) {
+        Map<String,String> mavenUris = new HashMap<String,String>();
+        guessMavenRUIFromIndex(jarFile, true, mavenUris);
+        
+        Set<MavenArtifact> artifacts = new HashSet<MavenArtifact>();
+        for(String uri: mavenUris.keySet()) {
+            MavenArtifact art = MavenUrlHelper.parseMvnUrl(uri);
+            if(art!=null) {
+                artifacts.add(art);
+            }
+        }
+        
+        needToDeploy.put(jarFile, artifacts);
     }
 
     private boolean canDeployFromCustomComponentFolder(String fileName) {

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/ShareComponentsLibsJob.java

@@ -0,0 +1,150 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.librariesmanager.model.service;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.IStatus;
+import org.eclipse.core.runtime.Status;
+import org.eclipse.core.runtime.jobs.Job;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.core.nexus.ArtifactRepositoryBean;
+import org.talend.core.nexus.IRepositoryArtifactHandler;
+import org.talend.core.nexus.RepositoryArtifactHandlerManager;
+import org.talend.core.nexus.TalendLibsServerManager;
+import org.talend.core.runtime.maven.MavenArtifact;
+import org.talend.librariesmanager.maven.MavenArtifactsHandler;
+import org.talend.librariesmanager.nexus.utils.ShareLibrariesUtil;
+
+/**
+ * created by bhe
+ */
+public class ShareComponentsLibsJob extends Job {
+
+    private MavenArtifactsHandler deployer;
+
+    private Map<File, Set<MavenArtifact>> needToDeploy;
+
+    public ShareComponentsLibsJob(String name, Map<File, Set<MavenArtifact>> needToDeploy, MavenArtifactsHandler deployer) {
+        super(name);
+        if (needToDeploy == null || deployer == null) {
+            throw new IllegalArgumentException("needToDeploy or deployer is null!");
+        }
+        this.deployer = deployer;
+        this.needToDeploy = needToDeploy;
+    }
+
+    @Override
+    protected IStatus run(IProgressMonitor monitor) {
+        deploy();
+        return Status.OK_STATUS;
+    }
+
+    private void deploy() {
+
+        // deploy needed jars for User and Exchange component providers
+        Map<String, List<MavenArtifact>> snapshotArtifactMap = new HashMap<String, List<MavenArtifact>>();
+        Map<String, List<MavenArtifact>> releaseArtifactMap = new HashMap<String, List<MavenArtifact>>();
+        if (!needToDeploy.isEmpty()) {
+
+            // collect groupId to search
+            Set<String> snapshotGroupIdSet = new HashSet<String>();
+            Set<String> releaseGroupIdSet = new HashSet<String>();
+            Set<Entry<File, Set<MavenArtifact>>> entries = needToDeploy.entrySet();
+
+            for (Entry<File, Set<MavenArtifact>> entry : entries) {
+                for (MavenArtifact art : entry.getValue()) {
+                    if (ShareLibrariesUtil.isSnapshotVersion(art.getVersion())) {
+                        snapshotGroupIdSet.add(art.getGroupId());
+                    } else {
+                        releaseGroupIdSet.add(art.getGroupId());
+                    }
+                }
+
+            }
+
+            // search on nexus to avoid deploy the jar many times
+            Map<File, Set<MavenArtifact>> shareFiles = new HashMap<File, Set<MavenArtifact>>();
+            ArtifactRepositoryBean customNexusServer = TalendLibsServerManager.getInstance().getCustomNexusServer();
+            IRepositoryArtifactHandler customerRepHandler = RepositoryArtifactHandlerManager
+                    .getRepositoryHandler(customNexusServer);
+            if (customerRepHandler != null) {
+
+                try {
+                    ShareLibrariesUtil.seachArtifacts(null, customerRepHandler, snapshotArtifactMap, releaseArtifactMap,
+                            snapshotGroupIdSet, releaseGroupIdSet);
+                } catch (Exception e1) {
+                    ExceptionHandler.process(e1);
+                }
+
+                for (Entry<File, Set<MavenArtifact>> entry : entries) {
+                    Set<MavenArtifact> toShareArtifacts = new HashSet<MavenArtifact>();
+
+                    for (MavenArtifact art : entry.getValue()) {
+                        boolean isSnapshotVersion = ShareLibrariesUtil.isSnapshotVersion(art.getVersion());
+                        String key = ShareLibrariesUtil.getArtifactKey(art, isSnapshotVersion);
+
+                        List<MavenArtifact> artifactList = null;
+                        boolean toShare = true;
+                        if (isSnapshotVersion) {
+                            artifactList = snapshotArtifactMap.get(key);
+                            if (artifactList != null && artifactList.size() > 0) {
+                                try {
+                                    if (ShareLibrariesUtil.isSameFileWithRemote(entry.getKey(), artifactList, customNexusServer,
+                                            customerRepHandler, isSnapshotVersion)) {
+                                        toShare = false;
+                                    }
+                                } catch (Exception e) {
+                                    ExceptionHandler.process(e);
+                                }
+                            }
+                        } else {
+                            artifactList = releaseArtifactMap.get(key);
+                            // skip checksum for release artifact.
+                            if (artifactList != null && artifactList.contains(art)) {
+                                toShare = false;
+                            }
+                        }
+                        if (toShare) {
+                            toShareArtifacts.add(art);
+                        }
+                    }
+
+                    if (!toShareArtifacts.isEmpty()) {
+                        shareFiles.put(entry.getKey(), toShareArtifacts);
+                    }
+                }
+
+                // share to remote
+                Set<Entry<File, Set<MavenArtifact>>> entriesToShare = shareFiles.entrySet();
+                for (Entry<File, Set<MavenArtifact>> entry : entriesToShare) {
+                    for (MavenArtifact art : entry.getValue()) {
+                        try {
+                            deployer.deploy(entry.getKey().getAbsoluteFile(), art);
+                        } catch (Exception e) {
+                            ExceptionHandler.process(e);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+}

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/nexus/utils/ShareLibrariesUtil.java

@@ -19,14 +19,18 @@ import java.util.Comparator;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.codehaus.jackson.map.util.ISO8601Utils;
+import org.eclipse.core.runtime.IProgressMonitor;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.core.nexus.ArtifactRepositoryBean;
 import org.talend.core.nexus.IRepositoryArtifactHandler;
 import org.talend.core.runtime.maven.MavenArtifact;
+import org.talend.core.runtime.maven.MavenUrlHelper;
+import org.talend.librariesmanager.i18n.Messages;
 
 public class ShareLibrariesUtil {
 
@@ -139,4 +143,56 @@ public class ShareLibrariesUtil {
         }
         return classifier;
     }
+
+    /**
+     * Search artifacts based on given snapshotGroupIdSet and releaseGroupIdSet from remote artifact repositories
+     * represented by artifactHandler, the search results are put to snapshotArtifactMap and releaseArtifactMap
+     */
+    public static void seachArtifacts(IProgressMonitor monitor, IRepositoryArtifactHandler artifactHandler,
+            Map<String, List<MavenArtifact>> snapshotArtifactMap, Map<String, List<MavenArtifact>> releaseArtifactMap,
+            Set<String> snapshotGroupIdSet, Set<String> releaseGroupIdSet) throws Exception {
+        if (artifactHandler != null) {
+            checkCancel(monitor);
+            List<MavenArtifact> searchResults = new ArrayList<MavenArtifact>();
+            for (String groupId : releaseGroupIdSet) {
+                searchResults = artifactHandler.search(groupId, null, null, true, false);
+                if (searchResults != null) {
+                    for (MavenArtifact result : searchResults) {
+                        checkCancel(monitor);
+                        ShareLibrariesUtil.putArtifactToMap(result, releaseArtifactMap, false);
+                    }
+                }
+            }
+            checkCancel(monitor);
+            for (String groupId : snapshotGroupIdSet) {
+                searchResults = artifactHandler.search(groupId, null, null, false, true);
+                if (searchResults != null) {
+                    for (MavenArtifact result : searchResults) {
+                        checkCancel(monitor);
+                        ShareLibrariesUtil.putArtifactToMap(result, snapshotArtifactMap, true);
+                    }
+                }
+            }
+        }
+    }
+
+    public static boolean isTalendLibraryGroupId(MavenArtifact artifact) {
+        if ("org.talend.libraries".equalsIgnoreCase(artifact.getGroupId())) {
+            return true;
+        }
+        return false;
+    }
+
+    public static boolean isSnapshotVersion(String version) {
+        if (version != null && version.toUpperCase().endsWith(MavenUrlHelper.VERSION_SNAPSHOT)) {
+            return true;
+        }
+        return false;
+    }
+
+    public static void checkCancel(IProgressMonitor monitor) throws InterruptedException {
+        if (monitor != null && monitor.isCanceled()) {
+            throw new InterruptedException(Messages.getString("ShareLibsJob.monitor.cancelled"));
+        }
+    }
 }

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/prefs/LibrariesManagerUtils.java

@@ -18,11 +18,16 @@ import java.util.List;
 import java.util.Set;
 
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.preferences.IEclipsePreferences;
+import org.eclipse.core.runtime.preferences.InstanceScope;
+import org.talend.commons.exception.ExceptionHandler;
+import org.talend.commons.utils.network.NetworkUtil;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.language.ECodeLanguage;
 import org.talend.core.model.general.ModuleNeeded;
 import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
 import org.talend.core.model.process.INode;
+import org.talend.core.prefs.ITalendCorePrefConstants;
 import org.talend.designer.core.IDesignerCoreService;
 import org.talend.librariesmanager.model.ModulesNeededProvider;
 
@@ -122,4 +127,15 @@ public class LibrariesManagerUtils {
         }
         return false;
     }
+
+    public static boolean shareLibsAtStartup() {
+        boolean ret = ITalendCorePrefConstants.NEXUS_SHARE_LIBS_DEFAULT;
+        try {
+            IEclipsePreferences node = InstanceScope.INSTANCE.getNode(NetworkUtil.ORG_TALEND_DESIGNER_CORE);
+            ret = node.getBoolean(ITalendCorePrefConstants.NEXUS_SHARE_LIBS, ITalendCorePrefConstants.NEXUS_SHARE_LIBS_DEFAULT);
+        } catch (Throwable e) {
+            ExceptionHandler.process(e);
+        }
+        return ret;
+    }
 }

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/ExtractMetaDataUtils.java

@@ -995,6 +995,12 @@ public class ExtractMetaDataUtils {
                     }else if(EDatabaseTypeName.SYBASEASE.getDisplayName().equals(dbType)
                             && EDatabaseVersion4Drivers.SYBASEIQ_16_SA.getVersionValue().equals(dbVersion)){
                         driverClassName = EDatabase4DriverClassName.SYBASEIQ_16_SA.getDriverClass();
+                    }else if(EDatabaseTypeName.GREENPLUM.getDisplayName().equals(dbType)
+                            && EDatabaseVersion4Drivers.GREENPLUM_PSQL.getVersionValue().equals(dbVersion)){
+                        driverClassName = EDatabase4DriverClassName.GREENPLUM_PSQL.getDriverClass();
+                    }else if(EDatabaseTypeName.GREENPLUM.getDisplayName().equals(dbType)
+                            && EDatabaseVersion4Drivers.GREENPLUM.getVersionValue().equals(dbVersion)){
+                        driverClassName = EDatabase4DriverClassName.GREENPLUM.getDriverClass();
                     }
                 }
             } else {

main/plugins/org.talend.model.migration/.classpath

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
-	<classpathentry kind="src" path="src"/>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
-	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
-	<classpathentry kind="output" path="bin"/>
-</classpath>

main/plugins/org.talend.model.migration/.project

@@ -1,34 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
-	<name>org.talend.model.migration</name>
-	<comment></comment>
-	<projects>
-	</projects>
-	<buildSpec>
-		<buildCommand>
-			<name>org.eclipse.jdt.core.javabuilder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
-		<buildCommand>
-			<name>org.eclipse.pde.ManifestBuilder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
-		<buildCommand>
-			<name>org.eclipse.pde.SchemaBuilder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
-		<buildCommand>
-			<name>org.eclipse.m2m.atl.adt.builder.atlBuilder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
-	</buildSpec>
-	<natures>
-		<nature>org.eclipse.m2m.atl.adt.builder.atlNature</nature>
-		<nature>org.eclipse.pde.PluginNature</nature>
-		<nature>org.eclipse.jdt.core.javanature</nature>
-	</natures>
-</projectDescription>

main/plugins/org.talend.model.migration/META-INF/MANIFEST.MF

@@ -1,18 +0,0 @@
-Manifest-Version: 1.0
-Bundle-ManifestVersion: 2
-Bundle-Name: %pluginName
-Bundle-SymbolicName: org.talend.model.migration;singleton:=true
-Bundle-Version: 7.4.1.qualifier
-Bundle-ClassPath: .
-Bundle-Vendor: %providerName
-Bundle-Localization: plugin
-Require-Bundle: org.talend.model,
- org.apache.log4j,
- org.eclipse.m2m.atl.core;visibility:=reexport,
- org.eclipse.m2m.atl.engine.emfvm,
- org.eclipse.m2m.atl.core.ui.vm,
- org.eclipse.m2m.atl.engine.vm,
- org.eclipse.m2m.atl.drivers.emf4atl,
- org.talend.commons.runtime
-Bundle-ActivationPolicy: lazy
-Export-Package: org.talend.model.migration

main/plugins/org.talend.model.migration/META-INF/eclipse.inf

@@ -1 +0,0 @@
-jarprocessor.exclude.children=true

main/plugins/org.talend.model.migration/build.properties

@@ -1,18 +0,0 @@
-
-# <copyright>
-# </copyright>
-#
-# $Id$
-
-bin.includes = .,\
-               META-INF/,\
-               plugin.xml,\
-               plugin.properties,\
-               migration_metamodel/,\
-               transformation/
-jars.compile.order = .
-source.. = src/
-output.. = bin/
-bin.excludes = transformation/development/,\
-               transformation/top_metadata400to410.atl,\
-               transformation/tos_metadata400to410.atl