APPINT-35490:App & API Integration - Studio Metrics gathering: build types & APIs usage (#6159) (#6162)

This reverts commit 11d3c32066.

main/plugins/org.talend.commons.runtime/src/org/talend/commons/utils/workbench/extensions/ExtensionImplementationProvider.java

@@ -12,16 +12,28 @@
 // ============================================================================
 package org.talend.commons.utils.workbench.extensions;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 
+import org.apache.commons.lang3.StringUtils;
 import org.eclipse.core.runtime.IConfigurationElement;
 import org.eclipse.core.runtime.IExtension;
 import org.eclipse.core.runtime.IExtensionPoint;
 import org.eclipse.core.runtime.IExtensionRegistry;
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.preferences.ConfigurationScope;
+import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.IllegalPluginConfigurationException;
 import org.talend.commons.i18n.internal.Messages;
+import org.talend.utils.json.JSONException;
+import org.talend.utils.json.JSONObject;
 
 /**
  * Utilities class uses to get implementation of extension points defined by plug-ins. <br/>
@@ -36,6 +48,10 @@ public abstract class ExtensionImplementationProvider<I> {
 
     private String plugInId;
 
+    public final static String FILE_FEATURES_INDEX = "extra_feature.index";
+
+    public final static String DROP_BUNDLE_INFO = "drop.bundle.info";
+
     /**
      * Default Constructor. Must not be used.
      */
@@ -166,9 +182,17 @@ public abstract class ExtensionImplementationProvider<I> {
         }
 
         IExtension[] extensions = pt.getExtensions();
-
+        Map<String, String> dropBundles = null;
+        try {
+            dropBundles = getDropBundleInfo();
+        } catch (IOException e) {
+            ExceptionHandler.process(e);
+        }
         for (IExtension extension : extensions) {
-
+            if (dropBundles != null && dropBundles.containsKey(extension.getNamespaceIdentifier())
+                    && StringUtils.isEmpty(dropBundles.get(extension.getNamespaceIdentifier()))) {
+                continue;
+            }
             if (plugInId == null || extension.getNamespaceIdentifier().equals(plugInId)) {
                 String configurationElementName = extensionPointLimiter.getConfigurationElementName();
                 if (configurationElementName != null) {
@@ -196,6 +220,32 @@ public abstract class ExtensionImplementationProvider<I> {
         return toReturn;
     }
 
+    /**********************************************************
+     * Copied from org.talend.commons.configurator
+     **********************************************************/
+    public Map<String, String> getDropBundleInfo() throws IOException {
+        File indexFile = new File(ConfigurationScope.INSTANCE.getLocation().toFile(), FILE_FEATURES_INDEX);
+        if (!indexFile.exists()) {
+            return Collections.emptyMap();
+        }
+        Map<String, String> dropInfoMap = new HashMap<>();
+        try {
+            String jsonStr = new String(Files.readAllBytes(indexFile.toPath()));
+            if (!jsonStr.isEmpty()) {
+                JSONObject obj = new JSONObject(jsonStr);
+                JSONObject dropInfo = obj.getJSONObject(DROP_BUNDLE_INFO);
+                Iterator<String> iterator = dropInfo.keys();
+                while (iterator.hasNext()) {
+                    String key = iterator.next();
+                    dropInfoMap.put(key, dropInfo.getString(key));
+                }
+            }
+        } catch (JSONException e) {
+            throw new IOException(e);
+        }
+        return dropInfoMap;
+    }
+
     /**
      * DOC amaumont Comment method "createAndAddImplementation".
      *

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ILibraryManagerService.java

@@ -16,6 +16,7 @@ import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -154,6 +155,8 @@ public interface ILibraryManagerService extends IService {
     public boolean contains(String jarName);
 
     public void clearCache();
+    
+    public void deployLibsFromCustomComponents(File componentFolder, List<ModuleNeeded> modulesNeeded);
 
     @Deprecated
     public Set<String> list(boolean withComponent, IProgressMonitor... monitorWrap);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/ConnParameterKeys.java

@@ -380,5 +380,6 @@ public class ConnParameterKeys {
     
     public static final String CONN_PARA_KEY_KNOX_DIRECTORY="CONN_PARA_KEY_KNOX_DIRECTORY";
 
+    public static final String CONN_PARA_KEY_KNOX_TIMEOUT="CONN_PARA_KEY_KNOX_TIMEOUT";
 
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/database/conn/version/EDatabaseVersion4Drivers.java

@@ -25,8 +25,8 @@ import org.talend.core.database.conn.DatabaseConnConstants;
 public enum EDatabaseVersion4Drivers {
     // access
     ACCESS_JDBC(new DbVersion4Drivers(EDatabaseTypeName.ACCESS, new String[] {
-            "jackcess-2.1.0.jar", "ucanaccess-2.0.9.5.jar", "commons-lang-2.6.jar", "commons-logging-1.1.1.jar", "hsqldb.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-            "jackcess-encrypt-2.1.0.jar", "bcprov-jdk15on-1.51.jar", "talend-ucanaccess-utils-1.0.0.jar" })),
+            "jackcess-2.1.12.jar", "ucanaccess-2.0.9.5.jar", "commons-lang-2.6.jar", "commons-logging-1.1.3.jar", "hsqldb.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+            "jackcess-encrypt-2.1.4.jar", "bcprov-jdk15on-1.69.jar", "talend-ucanaccess-utils-1.0.0.jar" })),
     ACCESS_2003(new DbVersion4Drivers(EDatabaseTypeName.ACCESS, "Access 2003", "Access_2003")), //$NON-NLS-1$ //$NON-NLS-2$
     ACCESS_2007(new DbVersion4Drivers(EDatabaseTypeName.ACCESS, "Access 2007", "Access_2007")), //$NON-NLS-1$ //$NON-NLS-2$
     // oracle
@@ -171,8 +171,10 @@ public enum EDatabaseVersion4Drivers {
     REDSHIFT(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT, "redshift", "REDSHIFT", //$NON-NLS-1$ //$NON-NLS-2$
             new String[]{ "redshift-jdbc42-no-awssdk-1.2.55.1083.jar", "antlr4-runtime-4.8-1.jar" })), //$NON-NLS-1$ //$NON-NLS-2$
     REDSHIFT_SSO(new DbVersion4Drivers(EDatabaseTypeName.REDSHIFT_SSO, "redshift sso", "REDSHIFT_SSO", //$NON-NLS-1$ //$NON-NLS-2$
-            new String[] { "redshift-jdbc42-no-awssdk-1.2.55.1083.jar", "antlr4-runtime-4.8-1.jar", "aws-java-sdk-1.11.848.jar", "jackson-core-2.10.1.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
-                    "jackson-databind-2.10.1.jar", "jackson-annotations-2.10.1.jar", "httpcore-4.4.11.jar", "httpclient-4.5.9.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$//$NON-NLS-4$
+            new String[] { "redshift-jdbc42-no-awssdk-1.2.55.1083.jar", "antlr4-runtime-4.8-1.jar", "aws-java-sdk-1.11.848.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+                    "jackson-core-2.13.4.jar", //$NON-NLS-1$
+                    "jackson-databind-2.13.4.2.jar", "jackson-annotations-2.13.4.jar", "httpcore-4.4.11.jar", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+                    "httpclient-4.5.9.jar", //$NON-NLS-1$
                     "joda-time-2.8.1.jar", "commons-logging-1.2.jar", "commons-codec-1.11.jar" })), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
 
     AMAZON_AURORA(new DbVersion4Drivers(EDatabaseTypeName.AMAZON_AURORA, "mysql-connector-java-5.1.49.jar")); //$NON-NLS-1$

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/HadoopClassLoaderFactory2.java

@@ -38,7 +38,10 @@ public class HadoopClassLoaderFactory2 {
     public static ClassLoader getHDFSClassLoader(String relatedClusterId, String distribution, String version, boolean useKrb) {
         return getClassLoader(relatedClusterId, EHadoopCategory.HDFS, distribution, version, useKrb);
     }
-
+    public static ClassLoader getHDFSKnoxClassLoader(String relatedClusterId, String distribution, String version, boolean useKrb) {
+        return HadoopClassLoaderFactory2.getClassLoader(relatedClusterId, EHadoopCategory.HDFS, distribution, version, useKrb,
+                IHadoopArgs.HDFS_ARG_KNOX);
+    }
     public static ClassLoader getMRClassLoader(String relatedClusterId, String distribution, String version, boolean useKrb) {
         return getClassLoader(relatedClusterId, EHadoopCategory.MAP_REDUCE, distribution, version, useKrb);
     }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/hadoop/IHadoopArgs.java

@@ -22,4 +22,6 @@ public interface IHadoopArgs {
 
     public static final String HIVE_ARG_STANDALONE = "STANDALONE"; //$NON-NLS-1$
 
+    public static final String HDFS_ARG_KNOX = "USE_KNOX"; //$NON-NLS-1$
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/context/ContextUtils.java

@@ -951,6 +951,14 @@ public class ContextUtils {
             params.add(param);
         }
 
+        public boolean remove(Item item, String param) {
+            Set<String> params = map.get(item);
+            if (params != null && params.contains(param)) {
+                return params.remove(param);
+            }
+            return false;
+        }
+
         @SuppressWarnings("unchecked")
         public Set<String> get(Item item) {
             Set<String> params = map.get(item);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/IMetadataConnection.java

@@ -194,6 +194,10 @@ public interface IMetadataConnection extends IMetadata {
     public String getContextName();
 
     public void setContextName(String contextName);
+    
+    public boolean isSupportNLS();
+    
+    public void setSupportNLS(boolean newSupportNLS);
 
     /**
      * Returns the value that you stored in the data collection by the key. Normally, it is like this key-value. For

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/builder/ConvertionHelper.java

@@ -254,6 +254,7 @@ public final class ConvertionHelper {
         result.setContentModel(connection.isContextMode());
         result.setContextId(sourceConnection.getContextId());
         result.setContextName(sourceConnection.getContextName());
+        result.setSupportNLS(sourceConnection.isSupportNLS());
         // handle oracle database connnection of general_jdbc.
         result.setSchema(getMeataConnectionSchema(result));
         convertOtherParameters(result, connection);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/builder/MetadataConnection.java

@@ -113,6 +113,7 @@ public class MetadataConnection implements IMetadataConnection {
 
     private String contextName;
 
+    private boolean supportNLS = false;
     // ~
 
     private String comment;
@@ -729,6 +730,14 @@ public class MetadataConnection implements IMetadataConnection {
     public void setContextName(String contextName) {
         this.contextName = contextName;
     }
+    
+    public boolean isSupportNLS() {
+        return supportNLS;
+    }
+    
+    public void setSupportNLS(boolean supportNLS) {
+        this.supportNLS = supportNLS;
+    }
 
     /*
      * (non-Javadoc)

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/metadata/designerproperties/RepositoryToComponentProperty.java

@@ -1225,6 +1225,11 @@ public class RepositoryToComponentProperty {
             return value2;
 
         }
+        
+        if(value.equals("SUPPORT_NLS")) {
+            return connection.isSupportNLS();
+        }
+        
         if (value.equals("CDC_TYPE_MODE")) { //$NON-NLS-1$
             return new Boolean(CDCTypeMode.LOG_MODE.getName().equals(connection.getCdcTypeMode()));
         }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/TalendPropertiesUtil.java

@@ -79,4 +79,8 @@ public final class TalendPropertiesUtil {
     public static String getProductApp() {
         return System.getProperty(PROD_APP);
     }
+
+    public static boolean isEnabledUseShortJobletName() {
+        return isEnabled("talend.job.build.useShortJobletName"); //$NON-NLS-1$
+    }
 }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/proposal/RoutinesFunctionProposal.java

@@ -78,6 +78,7 @@ public class RoutinesFunctionProposal implements IContentProposal {
         message += Messages.getString("RoutinesFunctionProposal.CreatedBy");
         message += Messages.getString("RoutinesFunctionProposal.ReturnType");
         message += Messages.getString("RoutinesFunctionProposal.VariableName");
+        message = message.replaceAll("\n", System.getProperty("line.separator", "\n")); // for display on Windows platform
 
         MessageFormat format = new MessageFormat(message);
         Object[] args = new Object[] { function.getDescription(),

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/token/TokenInforUtil.java

@@ -134,7 +134,7 @@ public final class TokenInforUtil {
                     targetArray = new JSONArray();
                     Map<String,List<JSONObject>> objectMap = new HashMap<String,List<JSONObject>>();
                     for (Object obj : data) {
-                        if((obj instanceof JSONObject) && ((JSONObject)obj).get("component_name")!=null){//$NON-NLS-1$
+                        if((obj instanceof JSONObject) && ((JSONObject)obj).has("component_name") && ((JSONObject)obj).get("component_name")!=null){//$NON-NLS-1$
                             List<JSONObject> dataList = new ArrayList<JSONObject>();
                             String componentName = (String) ((JSONObject)obj).get("component_name");//$NON-NLS-1$
                             if(objectMap.containsKey(componentName)){

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/RepositorySystemFactory.java

@@ -46,6 +46,8 @@ import org.talend.designer.maven.aether.util.TalendAetherProxySelector;
  */
 public class RepositorySystemFactory {
 
+    private static Boolean ignoreArtifactDescriptorRepositories;
+
     private static Map<LocalRepository, DefaultRepositorySystemSession> sessions = new HashMap<LocalRepository, DefaultRepositorySystemSession>();
 
     private static DefaultRepositorySystemSession newRepositorySystemSession(String localRepositoryPath)
@@ -61,6 +63,8 @@ public class RepositorySystemFactory {
             repositorySystemSession.setTransferListener(new ChainedTransferListener());
             repositorySystemSession.setRepositoryListener(new ChainedRepositoryListener());
             repositorySystemSession.setProxySelector(new TalendAetherProxySelector());
+            repositorySystemSession.setIgnoreArtifactDescriptorRepositories(
+                    RepositorySystemFactory.isIgnoreArtifactDescriptorRepositories());
             sessions.put(localRepo, repositorySystemSession);
         }
 
@@ -157,4 +161,13 @@ public class RepositorySystemFactory {
         doDeploy(content, pomFile, localRepository, repositoryId, repositoryUrl, userName, password, groupId, artifactId,
                 classifier, extension, version);
     }
+
+    public static boolean isIgnoreArtifactDescriptorRepositories() {
+        if (ignoreArtifactDescriptorRepositories == null) {
+            ignoreArtifactDescriptorRepositories = Boolean.valueOf(
+                    System.getProperty("talend.studio.aether.ignoreArtifactDescriptorRepositories", Boolean.TRUE.toString()));
+        }
+        return ignoreArtifactDescriptorRepositories;
+    }
+
 }

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/DynamicDistributionAetherUtils.java

@@ -63,6 +63,7 @@ import org.eclipse.m2e.core.MavenPlugin;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.designer.maven.aether.DummyDynamicMonitor;
 import org.talend.designer.maven.aether.IDynamicMonitor;
+import org.talend.designer.maven.aether.RepositorySystemFactory;
 import org.talend.designer.maven.aether.node.DependencyNode;
 import org.talend.designer.maven.aether.node.ExclusionNode;
 import org.talend.designer.maven.aether.selector.DynamicDependencySelector;
@@ -513,6 +514,7 @@ public class DynamicDistributionAetherUtils {
         LocalRepository localRepo = new LocalRepository(repositoryPath);
         session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo));
         session.setProxySelector(new TalendAetherProxySelector());
+        session.setIgnoreArtifactDescriptorRepositories(RepositorySystemFactory.isIgnoreArtifactDescriptorRepositories());
 
         updateDependencySelector(session, monitor);
 

main/plugins/org.talend.designer.maven.aether/src/main/java/org/talend/designer/maven/aether/util/MavenLibraryResolverProvider.java

@@ -34,7 +34,6 @@ import org.codehaus.plexus.PlexusContainerException;
 import org.eclipse.aether.DefaultRepositorySystemSession;
 import org.eclipse.aether.RepositorySystem;
 import org.eclipse.aether.RepositorySystemSession;
-import org.eclipse.aether.artifact.Artifact;
 import org.eclipse.aether.artifact.DefaultArtifact;
 import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
 import org.eclipse.aether.impl.DefaultServiceLocator;
@@ -57,6 +56,7 @@ import org.talend.core.nexus.ArtifactRepositoryBean;
 import org.talend.core.nexus.NexusConstants;
 import org.talend.core.nexus.TalendLibsServerManager;
 import org.talend.core.runtime.maven.MavenArtifact;
+import org.talend.designer.maven.aether.RepositorySystemFactory;
 
 public class MavenLibraryResolverProvider {
 
@@ -283,8 +283,9 @@ public class MavenLibraryResolverProvider {
         LocalRepository localRepo = new LocalRepository( /* "target/local-repo" */target);
         session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo));
         session.setProxySelector(new TalendAetherProxySelector());
+        session.setIgnoreArtifactDescriptorRepositories(RepositorySystemFactory.isIgnoreArtifactDescriptorRepositories());
 
-        return session;
+        return session; 
     }
 
     private String getLocalMVNRepository() {

main/plugins/org.talend.designer.maven.tos/resources/build/pom.xml

@@ -139,10 +139,15 @@
 			<version>1.21</version>
 		</dependency>
 		<dependency>
-            <groupId>org.apache-extras.beanshell</groupId>
-            <artifactId>bsh</artifactId>
-            <version>2.0b6</version>
-        </dependency>
+			<groupId>org.apache-extras.beanshell</groupId>
+			<artifactId>bsh</artifactId>
+			<version>2.0b6</version>
+		</dependency>
+		 <dependency>
+			<groupId>org.jboss.marshalling</groupId>
+			<artifactId>jboss-marshalling</artifactId>
+			<version>2.0.12.Final</version>
+		</dependency>
 	</dependencies>
 	<build>
 		<plugins>

main/plugins/org.talend.designer.maven.tos/resources/components/pom.xml

@@ -42,7 +42,7 @@
 			<dependency>
                 <groupId>org.json</groupId>
                 <artifactId>json</artifactId>
-                <version>20140107</version>
+                <version>20230227</version>
             </dependency>
             <dependency>
                 <groupId>com.fasterxml.jackson.dataformat</groupId>
@@ -151,6 +151,12 @@
 			<groupId>org.talend.components</groupId>
 			<artifactId>components-marklogic-runtime</artifactId>
 			<version>${components.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.apache.commons</groupId>
+                     <artifactId>commons-text</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
         <dependency>
             <groupId>org.talend.components</groupId>
@@ -210,6 +216,11 @@
 			<artifactId>commons-beanutils</artifactId>
 			<version>1.9.4</version>
 		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-text</artifactId>
+			<version>1.10.0</version>
+		</dependency>
 	</dependencies>
 	<build>
 		<plugins>

main/plugins/org.talend.designer.maven.tos/resources/pom.xml

@@ -20,7 +20,7 @@
         <module>zip/pom.xml</module>
     </modules>
     <properties>
-        <m2.fasterxml.jackson.version>2.13.2</m2.fasterxml.jackson.version>
+        <m2.fasterxml.jackson.version>2.13.4</m2.fasterxml.jackson.version>
         <jackson-codehaus.version>1.9.16-TALEND</jackson-codehaus.version>
     </properties>
 </project>

main/plugins/org.talend.designer.maven.tos/resources/tacokit/pom.xml

@@ -10,7 +10,7 @@
 	<artifactId>studio-tacokit-dependencies</artifactId>
 	<packaging>pom</packaging>
 	<properties>
-		<tacokit.components.version>1.27.13</tacokit.components.version>
+		<tacokit.components.version>1.27.22</tacokit.components.version>
 	</properties>
 	<repositories>
 		<repository>

main/plugins/org.talend.designer.maven.tos/resources/tcompv1/pom.xml

@@ -11,7 +11,7 @@
     <packaging>pom</packaging>
 
     <properties>
-        <tcomp.version>1.38.6</tcomp.version>
+        <tcomp.version>1.38.9</tcomp.version>
         <slf4j.version>1.7.32</slf4j.version>
         <log4j2.version>2.17.1</log4j2.version>
         <reload4j.version>1.2.19</reload4j.version>

main/plugins/org.talend.libraries.apache.common/.classpath

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
+	<classpathentry exported="true" kind="lib" path="lib/commons-text-1.10.0.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/commons-pool2-2.4.2.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/commons-validator-1.5.1.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/commons-math3-3.3.jar"/>
@@ -8,6 +9,5 @@
 	<classpathentry exported="true" kind="lib" path="lib/commons-digester-2.1.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/commons-cli-2.0-SNAPSHOT.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/commons-codec-1.15.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/commons-text-1.1.jar"/>
 	<classpathentry kind="output" path="bin"/>
 </classpath>

main/plugins/org.talend.libraries.apache.common/META-INF/MANIFEST.MF

@@ -10,7 +10,7 @@ Bundle-ClassPath: .,
  lib/commons-math3-3.3.jar,
  lib/commons-validator-1.5.1.jar,
  lib/commons-pool2-2.4.2.jar,
- lib/commons-text-1.1.jar
+ lib/commons-text-1.10.0.jar
 Export-Package: org.apache.commons.cli2,
  org.apache.commons.cli2.builder,
  org.apache.commons.cli2.commandline,

main/plugins/org.talend.libraries.apache.common/build.properties

@@ -7,4 +7,4 @@ bin.includes = META-INF/,\
                lib/commons-math3-3.3.jar,\
                lib/commons-validator-1.5.1.jar,\
                lib/commons-pool2-2.4.2.jar,\
-               lib/commons-text-1.1.jar
+               lib/commons-text-1.10.0.jar

main/plugins/org.talend.libraries.apache.common/pom.xml

@@ -62,6 +62,11 @@
 									<artifactId>commons-collections</artifactId>
 									<version>3.2.2</version>
 								</artifactItem>
+								<artifactItem>
+									<groupId>org.apache.commons</groupId>
+									<artifactId>commons-text</artifactId>
+									<version>1.10.0</version>
+								</artifactItem>
 								<artifactItem>
 									<groupId>commons-io</groupId>
 									<artifactId>commons-io</artifactId>

main/plugins/org.talend.libraries.apache.cxf/.classpath

@@ -6,23 +6,23 @@
 	<classpathentry exported="true" kind="lib" path="lib/wsdl4j-1.6.3.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/istack-commons-runtime-3.0.12.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/jaxb-runtime-2.3.4.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/woodstox-core-6.2.6.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-core-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-bindings-soap-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-bindings-xml-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-databinding-jaxb-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-features-clustering-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-frontend-jaxrs-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-frontend-jaxws-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-frontend-simple-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-rs-client-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-security-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-security-saml-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-transports-http-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-ws-addr-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-wsdl-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-ws-policy-3.4.7.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-ws-security-3.4.7.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/woodstox-core-6.4.0.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-core-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-bindings-soap-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-bindings-xml-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-databinding-jaxb-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-features-clustering-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-frontend-jaxrs-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-frontend-jaxws-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-frontend-simple-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-rs-client-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-security-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-security-saml-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-transports-http-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-ws-addr-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-wsdl-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-ws-policy-3.5.5.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/cxf-rt-ws-security-3.5.5.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/jakarta.activation-1.2.2.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/jakarta.activation-api-1.2.2.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/jakarta.annotation-api-1.3.5.jar"/>

main/plugins/org.talend.libraries.apache.cxf/META-INF/MANIFEST.MF

@@ -5,22 +5,22 @@ Bundle-SymbolicName: org.talend.libraries.apache.cxf;singleton:=true
 Bundle-Version: 7.3.1.qualifier
 Bundle-ActivationPolicy: lazy
 Bundle-ClassPath: .,
- lib/cxf-core-3.4.7.jar,
- lib/cxf-rt-bindings-soap-3.4.7.jar,
- lib/cxf-rt-bindings-xml-3.4.7.jar,
- lib/cxf-rt-databinding-jaxb-3.4.7.jar,
- lib/cxf-rt-features-clustering-3.4.7.jar,
- lib/cxf-rt-frontend-jaxrs-3.4.7.jar,
- lib/cxf-rt-frontend-jaxws-3.4.7.jar,
- lib/cxf-rt-frontend-simple-3.4.7.jar,
- lib/cxf-rt-rs-client-3.4.7.jar,
- lib/cxf-rt-security-3.4.7.jar,
- lib/cxf-rt-security-saml-3.4.7.jar,
- lib/cxf-rt-transports-http-3.4.7.jar,
- lib/cxf-rt-ws-addr-3.4.7.jar,
- lib/cxf-rt-wsdl-3.4.7.jar,
- lib/cxf-rt-ws-security-3.4.7.jar,
- lib/cxf-rt-ws-policy-3.4.7.jar,
+ lib/cxf-core-3.5.5.jar,
+ lib/cxf-rt-bindings-soap-3.5.5.jar,
+ lib/cxf-rt-bindings-xml-3.5.5.jar,
+ lib/cxf-rt-databinding-jaxb-3.5.5.jar,
+ lib/cxf-rt-features-clustering-3.5.5.jar,
+ lib/cxf-rt-frontend-jaxrs-3.5.5.jar,
+ lib/cxf-rt-frontend-jaxws-3.5.5.jar,
+ lib/cxf-rt-frontend-simple-3.5.5.jar,
+ lib/cxf-rt-rs-client-3.5.5.jar,
+ lib/cxf-rt-security-3.5.5.jar,
+ lib/cxf-rt-security-saml-3.5.5.jar,
+ lib/cxf-rt-transports-http-3.5.5.jar,
+ lib/cxf-rt-ws-addr-3.5.5.jar,
+ lib/cxf-rt-wsdl-3.5.5.jar,
+ lib/cxf-rt-ws-security-3.5.5.jar,
+ lib/cxf-rt-ws-policy-3.5.5.jar,
  lib/istack-commons-runtime-3.0.12.jar,
  lib/jakarta.activation-1.2.2.jar,
  lib/jakarta.activation-api-1.2.2.jar,
@@ -35,7 +35,7 @@ Bundle-ClassPath: .,
  lib/stax2-api-4.2.1.jar,
  lib/txw2-2.3.4.jar,
  lib/xmlschema-core-2.2.5.jar,
- lib/woodstox-core-6.2.6.jar,
+ lib/woodstox-core-6.4.0.jar,
  lib/wsdl4j-1.6.3.jar
 Export-Package: javax.jws,
  javax.ws.rs,

main/plugins/org.talend.libraries.apache.cxf/pom.xml

@@ -11,7 +11,7 @@
   <packaging>eclipse-plugin</packaging>
   
   <properties>
-    <cxf.version>3.4.7</cxf.version>
+    <cxf.version>3.5.5</cxf.version>
   </properties>
   
   <repositories>
@@ -170,7 +170,7 @@
 		<dependency>
 			<groupId>com.fasterxml.woodstox</groupId>
 			<artifactId>woodstox-core</artifactId>
-			<version>6.2.6</version>
+			<version>6.4.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.ws.xmlschema</groupId>

main/plugins/org.talend.libraries.jdbc.hsql/licenses/license.txt

@@ -6,9 +6,42 @@ COPYRIGHTS AND LICENSES
 
 ORIGINAL LICENSE (a.k.a. "hypersonic_lic.txt")
 
-For content, code, and products originally developed by Thomas Mueller and the Hypersonic SQL Group:
+For work developed by the HSQL Development Group:
 
-Copyright (c) 1995-2000 by the Hypersonic SQL Group.
+Copyright (c) 2001-2022, The HSQL Development Group
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+Neither the name of the HSQL Development Group nor the names of its
+contributors may be used to endorse or promote products derived from this
+software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+
+For work originally developed by the Hypersonic SQL Group:
+
+Copyright (c) 1995-2000, The Hypersonic SQL Group.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -37,12 +70,12 @@ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-This software consists of voluntary contributions made by many individuals on behalf of the
-Hypersonic SQL Group.
+This software consists of voluntary contributions made by many individuals
+on behalf of the Hypersonic SQL Group.
 
 For work added by the HSQL Development Group (a.k.a. hsqldb_lic.txt):
 
-Copyright (c) 2001-2005, The HSQL Development Group
+Copyright (c) 2001-2022, The HSQL Development Group
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

main/plugins/org.talend.libraries.jdbc.hsql/pom.xml

@@ -9,4 +9,33 @@
   </parent>
   <artifactId>org.talend.libraries.jdbc.hsql</artifactId>
   <packaging>eclipse-plugin</packaging>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>copy</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>copy</goal>
+            </goals>
+            <configuration>
+              <artifactItems>
+                <artifactItem>
+                  <groupId>org.hsqldb</groupId>
+                  <artifactId>hsqldb</artifactId>
+                  <version>2.7.1</version>
+                  <classifier>jdk8</classifier>
+                  <outputDirectory>${project.basedir}/lib</outputDirectory>
+                  <destFileName>hsqldb.jar</destFileName>
+                </artifactItem>
+              </artifactItems>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
 </project>

main/plugins/org.talend.libraries.persist.lookup/.classpath

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
-	<classpathentry exported="true" kind="lib" path="lib/advancedPersistentLookupLib-1.2.jar"/>
+	<classpathentry exported="true" kind="lib" path="lib/advancedPersistentLookupLib-1.4.jar"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
 	<classpathentry kind="output" path="bin"/>

main/plugins/org.talend.libraries.persist.lookup/META-INF/MANIFEST.MF

@@ -3,8 +3,8 @@ Bundle-ManifestVersion: 2
 Bundle-Name: org.talend.libraries.persist.lookup
 Bundle-SymbolicName: org.talend.libraries.persist.lookup
 Bundle-Version: 7.3.1.qualifier
-Bundle-ClassPath: lib/advancedPersistentLookupLib-1.2.jar,
- .
+Bundle-ClassPath: .,
+ lib/advancedPersistentLookupLib-1.4.jar
 Export-Package: org.talend.commons.utils.data.map,
  org.talend.commons.utils.time,
  org.talend.core.model.process,

main/plugins/org.talend.libraries.persist.lookup/build.properties

@@ -2,4 +2,6 @@ source.. = src/
 output.. = bin/
 bin.includes = META-INF/,\
                .,\
+               lib/advancedPersistentLookupLib-1.4.jar,\
+			   lib/advancedPersistentLookupLib-1.3.jar,\
                lib/advancedPersistentLookupLib-1.2.jar

main/plugins/org.talend.libraries.persist.lookup/export_advancedPersistentLookupLib.jardesc

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <jardesc>
-    <jar path="D:/studio_code/tcommon-studio-se/main/plugins/org.talend.libraries.persist.lookup/lib/advancedPersistentLookupLib-1.2.jar"/>
+    <jar path="D:/studio_code/tcommon-studio-se/main/plugins/org.talend.libraries.persist.lookup/lib/advancedPersistentLookupLib-1.4.jar"/>
     <options buildIfNeeded="true" compress="true" descriptionLocation="/org.talend.libraries.persist.lookup/export_advancedPersistentLookupLib.jardesc" exportErrors="true" exportWarnings="true" includeDirectoryEntries="false" overwrite="true" saveDescription="false" storeRefactorings="false" useSourceFolders="false"/>
     <storedRefactorings deprecationInfo="true" structuralOnly="false"/>
     <selectedProjects/>

main/plugins/org.talend.librariesmanager/.classpath

@@ -2,6 +2,7 @@
 <classpath>
 	<classpathentry kind="src" path="src/main/java"/>
 	<classpathentry kind="src" path="resources/java"/>
+	<classpathentry exported="true" kind="lib" path="lib/jboss-marshalling-2.0.12.Final.jar"/>
 	<classpathentry kind="lib" path="lib/crypto-utils.jar"/>
 	<classpathentry kind="lib" path="lib/slf4j-api-1.7.25.jar"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>

main/plugins/org.talend.librariesmanager/META-INF/MANIFEST.MF

@@ -28,7 +28,8 @@ Eclipse-LazyStart: true
 Bundle-ClassPath: .,
  lib/crypto-utils.jar,
  lib/slf4j-api-1.7.25.jar
-Export-Package: org.talend.librariesmanager.emf.librariesindex,
+Export-Package: org.jboss.marshalling,
+ org.talend.librariesmanager.emf.librariesindex,
  org.talend.librariesmanager.librarydata,
  org.talend.librariesmanager.maven,
  org.talend.librariesmanager.model,
@@ -40,3 +41,5 @@ Export-Package: org.talend.librariesmanager.emf.librariesindex,
 Import-Package: org.eclipse.emf.ecore.xmi.impl,
  org.talend.osgi.hook.notification
 Eclipse-BundleShape: dir
+Bundle-ClassPath: lib/jboss-marshalling-2.0.12.Final.jar,
+ .

main/plugins/org.talend.librariesmanager/plugin.xml

@@ -69,6 +69,12 @@
                name="crypto-utils-0.31.12.jar">
          </library>
       </systemRoutine>
+      <systemRoutine
+            name="IPersistableLookupRow">
+         <library
+               name="mvn:org.jboss.marshalling/jboss-marshalling/2.0.12.Final">
+         </library>
+      </systemRoutine>
    </extension>
    <extension
          point="org.talend.core.runtime.artifact_handler">

main/plugins/org.talend.librariesmanager/pom.xml

@@ -51,6 +51,11 @@
 									<version>1.7.25</version>
 									<overWrite>true</overWrite>
 								</artifactItem>
+								<artifactItem>
+									<groupId>org.jboss.marshalling</groupId>
+									<artifactId>jboss-marshalling</artifactId>
+									<version>2.0.12.Final</version>
+								</artifactItem>
 							</artifactItems>
 						</configuration>
 					</execution>

main/plugins/org.talend.librariesmanager/resources/java/routines/TalendDate.java

@@ -467,13 +467,13 @@ public class TalendDate {
      *
      * {Category} TalendDate
      *
-     * {param} String("") string : date represent in string
+     * {param} String("2008/11/24 12:15:25") string : date represent in string
      *
-     * {param} String("yyyy-MM-dd") pattern : date pattern
+     * {param} String("yyyy/MM/dd HH:mm:ss") pattern : date pattern
      *
-     * {param} int(addValue) nb : the added value
+     * {param} int(5) nb : the added value
      *
-     * {param} date("MM") dateType : the part to add
+     * {param} String("dd") dateType : the part to add
      *
      * {examples}
      *
@@ -1216,8 +1216,19 @@ public class TalendDate {
     }
 
     /**
-     * format date to mssql 2008 type datetimeoffset ISO 8601 string with local time zone format string : yyyy-MM-dd
-     * HH:mm:ss.SSSXXX(JDK7 support it)
+     * Format date to mssql 2008 type datetimeoffset ISO 8601 string with local time zone format string : yyyy-MM-dd
+     * HH:mm:ss.SSSXXX (JDK7 support it)
+     *
+     * @param date the time value to be formatted into a time string.
+     * @return the formatted time string.
+     *
+     * {talendTypes} String
+     *
+     * {Category} TalendDate
+     *
+     * {param} date(new Date()) date : the time value to be formatted into a time string
+     *
+     * {example} formatDatetimeoffset(new Date()) #
      */
     public static String formatDatetimeoffset(Date date) {
         String dateString = formatDate("yyyy-MM-dd HH:mm:ss.SSSZ", date);// keep the max precision in java
@@ -1346,14 +1357,28 @@ public class TalendDate {
     }
 
     /**
-     *
+     * Convert a formatted string to date
+     * 
      * @param string Must be a string datatype. Passes the values that you want to convert.
      * @param format Enter a valid TO_DATE format string. The format string must match the parts of the string argument
-     * 		  default formate is "MM/DD/yyyy HH:mm:ss.sss" if not specified.
+     * default format is "MM/DD/yyyy HH:mm:ss.sss" if not specified.
+     * 
      * @return Date
      * @throws ParseException
-     * {example} TO_DATE("1464576463231", "J") #Mon May 30 10:47:43 CST 2016
-     * {example} TO_DATE("2015-11-21 13:23:45","yyyy-MM-dd HH:mm:ss") #Sat Nov 21 13:23:45 CST 2015
+     * 
+     * {talendTypes} Date
+     *
+     * {Category} TalendDate
+     * 
+     * {param} String("2015-11-21 13:23:45") string : string Must be a string datatype. Passes the values that you want
+     * to convert.
+     *
+     * {param} String("yyyy-MM-dd HH:mm:ss") format : Enter a valid TO_DATE format string. The format string must match
+     * the parts of the string argument default format is "MM/DD/yyyy HH:mm:ss.sss" if not specified.
+     *
+     *
+     * {example} TO_DATE("1464576463231", "J") #Mon May 30 10:47:43 CST 2016 {example} TO_DATE("2015-11-21
+     * 13:23:45","yyyy-MM-dd HH:mm:ss") #Sat Nov 21 13:23:45 CST 2015
      *
      */
     public static Date TO_DATE(String string, String format) throws ParseException {
@@ -1374,6 +1399,24 @@ public class TalendDate {
 
     }
 
+    /**
+     * Convert a formatted string to date with default format as ""MM/DD/yyyy HH:mm:ss.sss"
+     * 
+     * @param string Must be a string datatype. Passes the values that you want to convert.
+     * @return Date
+     * @throws ParseException
+     * 
+     * {talendTypes} Date
+     *
+     * {Category} TalendDate
+     * 
+     * {param} String("11/21/2015 13:23:45.111") string : string Must be a string datatype. Passes the values that you
+     * want to convert.
+     *
+     * {example} TO_DATE("11/21/2015 13:23:45.111") #Sat Nov 21 13:23:45.111 CST 2015
+     *
+     */
+
     public static Date TO_DATE(String string) throws ParseException {
         return TO_DATE(string, null);
     }
@@ -1410,13 +1453,25 @@ public class TalendDate {
     }
 
     /**
-     *
-     * @param date  Passes the values you want to change
+     * Add values to the specified portion of the date
+     * 
+     * @param date Passes the values you want to change
      * @param format A format string specifying the portion of the date value you want to change.For example, 'mm'.
-     * @param amount An integer value specifying the amount of years, months, days, hours,
-     * 				and so on by which you want to change the date value.
-     * @return Date  NULL if a null value is passed as an argument to the function.
+     * @param amount An integer value specifying the amount of years, months, days, hours, and so on by which you want
+     * to change the date value.
+     * @return Date NULL if a null value is passed as an argument to the function.
      * @throws ParseException
+     * 
+     * {talendTypes} Date
+     *
+     * {Category} TalendDate
+     * 
+     * {param} Date(new Date()) date :
+     * 
+     * {param} String("HH") format :
+     * 
+     * {param} int(2) amount :
+     * 
      * {example} ADD_TO_DATE(new Date(1464576463231l), "HH",2) #Mon May 30 12:47:43 CST 2016
      */
     public static Date ADD_TO_DATE(Date date, String format, int amount) throws ParseException{
@@ -1485,10 +1540,21 @@ public class TalendDate {
     }
 
     /**
+     * Convert a Date to a formatted character string.
      *
-     * @param date  Date/Time datatype. Passes the date values you want to convert to character strings.
-     * @param format   Enter a valid TO_CHAR format string. The format string defines the format of the return value,
-     * @return  String.   NULL if a value passed to the function is NULL.
+     * @param date the date value you want to convert to character strings.
+     * @param format the format of the return value,
+     * @return String. NULL if a value passed to the function is NULL.
+     *
+     * {talendTypes} String
+     *
+     * {Category} TalendDate
+     *
+     * {param} Date(new Date()) date : the date value you want to convert to character strings.
+     * 
+     * {param} String("MM/DD/YYYY HH24:MI:SS") format : the format of the return value,
+     *
+     * {example} TO_CHAR(new Date(),"MM/DD/YYYY HH24:MI:SS") #
      */
 
     public static String TO_CHAR(Date date, String format) {

main/plugins/org.talend.librariesmanager/resources/java/routines/system/GetJarsToRegister.java

@@ -1,6 +1,6 @@
 // ============================================================================
 //
-// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
+// Copyright (C) 2006-2021 Talend Inc. - www.talend.com
 //
 // This source code is available under agreement available at
 // %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
@@ -14,7 +14,6 @@ package routines.system;
 
 import java.io.IOException;
 import java.io.ObjectInputStream;
-import java.util.HashMap;
 import java.util.List;
 
 import org.dom4j.Element;
@@ -110,26 +109,31 @@ public class GetJarsToRegister {
     private String addLibsPath(String line, java.util.Map<String, String> crcMap) {
         for (java.util.Map.Entry<String, String> entry : crcMap.entrySet()) {
             line = adaptLibPaths(line, entry);
+            if (new java.io.File(line).exists()) {
+                break;
+            }
         }
         return line;
     }
 
     private String adaptLibPaths(String line, java.util.Map.Entry<String, String> entry) {
+        line = line.replace("\\", "/");
         String jarName = entry.getValue();
         String crc = entry.getKey();
         String libStringFinder = "../lib/" + jarName;
         String libStringFinder2 = "./" + jarName; // for the job jar itself.
+        String replacement = "../../../cache/lib/" + crc + "/" + jarName;
 
         if (line.contains(libStringFinder)) {
-            line = line.replace(libStringFinder, "../../../cache/lib/" + crc + "/" + jarName);
+            line = line.replace(libStringFinder, replacement);
         } else if (line.toLowerCase().contains(libStringFinder2)) {
-            line = line.toLowerCase().replace(libStringFinder2, "../../../cache/lib/" + crc + "/" + jarName);
-        } else if (line.toLowerCase().equals(jarName)) {
-            line = "../../../cache/lib/" + crc + "/" + jarName;
+            line = line.toLowerCase().replace(libStringFinder2, replacement);
+        } else if (line.equalsIgnoreCase(jarName)) {
+            line = replacement;
         } else if (line.contains(":$ROOT_PATH/" + jarName + ":")) {
-            line = line.replace(":$ROOT_PATH/" + jarName + ":", ":$ROOT_PATH/../../../cache/lib/" + crc + "/" + jarName + ":");
+            line = line.replace(":$ROOT_PATH/" + jarName + ":", ":$ROOT_PATH/" + replacement + ":");
         } else if (line.contains(";" + jarName + ";")) {
-            line = line.replace(";" + jarName + ";", ";../../../cache/lib/" + crc + "/" + jarName + ";");
+            line = line.replace(";" + jarName + ";", ";" + replacement + ";");
         }
         return line;
     }

main/plugins/org.talend.librariesmanager/resources/java/routines/system/IPersistableLookupRow.java

@@ -5,6 +5,9 @@ import java.io.DataOutputStream;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 
+import org.jboss.marshalling.Marshaller;
+import org.jboss.marshalling.Unmarshaller;
+
 public interface IPersistableLookupRow<R> {
 
     public void writeKeysData(ObjectOutputStream out);
@@ -19,4 +22,25 @@ public interface IPersistableLookupRow<R> {
 
     public void copyKeysDataTo(R other);
 
+    default public void writeKeysData(Marshaller marshaller){
+        //sub-class need to override this method
+        throw new UnsupportedOperationException("Method need to be override");
+    }
+
+    default public void readKeysData(Unmarshaller in){
+        throw new UnsupportedOperationException("Method need to be override");
+    }
+
+    default public void writeValuesData(DataOutputStream dataOut, Marshaller objectOut){
+        throw new UnsupportedOperationException("Method need to be override");
+    }
+
+    default public void readValuesData(DataInputStream dataIn, Unmarshaller objectIn){
+        throw new UnsupportedOperationException("Method need to be override");
+    }
+
+    default public boolean supportMarshaller(){
+        //Override this method to return true after implement the Jboss methods above
+        return false;
+    }
 }

main/plugins/org.talend.librariesmanager/resources/java/routines/system/IPersistableRow.java

@@ -3,10 +3,28 @@ package routines.system;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 
+import org.jboss.marshalling.Marshaller;
+import org.jboss.marshalling.Unmarshaller;
+
 public interface IPersistableRow<R> {
 
     public void writeData(ObjectOutputStream out);
 
     public void readData(ObjectInputStream in);
 
+
+    default public void writeData(Marshaller marshaller){
+        //sub-class need to override this method
+        throw new UnsupportedOperationException("Method need to be override");
+    }
+
+    default public void readData(Unmarshaller in){
+        throw new UnsupportedOperationException("Method need to be override");
+    }
+
+    default public boolean supportJboss(){
+        //Override this method to return true after implement the Jboss methods above
+        return false;
+    }
+
 }

main/plugins/org.talend.librariesmanager/resources/java/routines/system/ResumeUtil.java

@@ -520,9 +520,13 @@ public class ResumeUtil {
 
         private String lineSeparator = System.getProperty("line.separator");
         
-        private int capibility = 2 << 22; //8M
+        private final int capibility = 2 << 22; //8M
+        
+        private final int FLUSH_FACTOR = 6 *1024 *1024; //6M
+        
+        private final int SUBSTRING_SIZE = 2 << 20; //2M
+        
         
-        private int FLUSH_FACTOR = 6 *1024 *1024; //6M
         
 
         public SimpleCsvWriter(FileChannel channel) {
@@ -553,6 +557,16 @@ public class ResumeUtil {
                 content = replace(content, "" + TextQualifier, "" + TextQualifier + TextQualifier);
             }
             
+            if (content.length() > SUBSTRING_SIZE) { //2M
+                int index = 0;
+                for (; content.length() - index > SUBSTRING_SIZE; index += SUBSTRING_SIZE) {
+                    flush(true);
+                    final String substring = content.substring(index, index + SUBSTRING_SIZE);
+                    buf.put(substring.getBytes());
+                }
+                content = content.substring(index);
+            }
+            
             byte[] contentByte = content.getBytes();
             if(contentByte.length > capibility - buf.position()) {
             	flush(true);

main/plugins/org.talend.librariesmanager/src/main/java/org/talend/librariesmanager/model/service/LocalLibraryManager.java

@@ -1319,11 +1319,81 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
         saveMavenIndex(mavenURIMap, monitorWrap);
         savePlatfromURLIndex(platformURLMap, monitorWrap);
 
-        if (service != null) {
-            deployLibsFromCustomComponents(service, platformURLMap);
-        }
         return mavenURIMap;
     }
+    
+    public void deployLibsFromCustomComponents(File componentFolder, List<ModuleNeeded> modulesNeeded) {
+        if (modulesNeeded == null || modulesNeeded.isEmpty()) {
+            return;
+        }
+        Map<File, Set<MavenArtifact>> needToDeploy = new HashMap<File, Set<MavenArtifact>>();
+        modulesNeeded.forEach(module -> {
+            if (module != null) {
+                boolean needDeploy = false;
+                String mvnUri = module.getMavenUri();
+                String jarPathFromMaven = getJarPathFromMaven(StringUtils.isNotBlank(mvnUri) ? mvnUri : module.getModuleName());
+                if (StringUtils.isBlank(jarPathFromMaven)) {
+                    needDeploy = true;
+                } else {
+                    File jarFromMaven = new File(jarPathFromMaven);
+                    if (!jarFromMaven.exists()) {
+                        needDeploy = true;
+                    }
+                }
+
+                if (needDeploy) {
+                    File deployFile = getDeployJarFileByModule(componentFolder, module);
+                    if (deployFile != null) {
+
+                        install(deployFile, mvnUri, false, true, null);
+
+                        if (needToDeploy.get(deployFile) == null) {
+                            needToDeploy.put(deployFile, new HashSet<MavenArtifact>());
+                        }
+                        if (StringUtils.isNotBlank(mvnUri)) {
+                            MavenArtifact mavenArtifact = MavenUrlHelper.parseMvnUrl(mvnUri);
+                            needToDeploy.get(deployFile).add(mavenArtifact);
+                        } else {
+                            Map<String, String> sourceAndMavenUri = new HashMap<>();
+                            guessMavenRUIFromIndex(deployFile, true, sourceAndMavenUri);
+                            Set<MavenArtifact> MavenArtifactSet = new HashSet<MavenArtifact>();
+                            sourceAndMavenUri.keySet().forEach(mavenUri -> {
+                                if (StringUtils.isNotBlank(mvnUri)) {
+                                    MavenArtifactSet.add(MavenUrlHelper.parseMvnUrl(mavenUri));
+                                }
+                            });
+                            needToDeploy.get(deployFile).addAll(MavenArtifactSet);
+                        }
+                    }
+                }
+            }
+        });
+
+        if (!needToDeploy.isEmpty()) {
+            ShareComponentsLibsJob shareJob = new ShareComponentsLibsJob(
+                    Messages.getString("LocalLibraryManager.shareLibsForCustomponents"), needToDeploy, deployer);
+            shareJob.schedule();
+        }
+    }
+
+    private File getDeployJarFileByModule(File componentFolder, ModuleNeeded module) {
+        String mvnUri = module.getMavenUri();
+        if (StringUtils.isNotBlank(mvnUri)) {
+            MavenArtifact mavenArtifact = MavenUrlHelper.parseMvnUrl(mvnUri);
+            String fileName = mavenArtifact.getFileName();
+            File jarFile = new File(componentFolder, fileName);
+            if (jarFile.exists()) {
+                return jarFile;
+            }
+
+        }
+        // try module name
+        File jarFile = new File(componentFolder, module.getModuleName());
+        if (jarFile.exists()) {
+            return jarFile;
+        }
+        return null;
+    }
 
     /**
      *
@@ -1382,86 +1452,6 @@ public class LocalLibraryManager implements ILibraryManagerService, IChangedLibr
 		return false;
 	}
 
-    private void deployLibsFromCustomComponents(IComponentsService service, Map<String, String> platformURLMap) {
-        boolean deployToRemote = true;
-        if (!LibrariesManagerUtils.shareLibsAtStartup()) {
-            log.info("Skip deploying libs from custom components");
-            deployToRemote = false;
-        }
-
-        Map<File, Set<MavenArtifact>> needToDeploy = new HashMap<File, Set<MavenArtifact>>();
-        List<ComponentProviderInfo> componentsFolders = service.getComponentsFactory().getComponentsProvidersInfo();
-        for (ComponentProviderInfo providerInfo : componentsFolders) {
-            String id = providerInfo.getId();
-            try {
-                File file = new File(providerInfo.getLocation());
-                if (isExtComponentProvider(id)) {
-                    if (file.isDirectory()) {
-                        List<File> jarFiles = FilesUtils.getJarFilesFromFolder(file, null);
-                        if (jarFiles.size() > 0) {
-                            for (File jarFile : jarFiles) {
-                                String name = jarFile.getName();
-                                if (!canDeployFromCustomComponentFolder(name) || platformURLMap.get(name) != null) {
-                                    continue;
-                                }
-
-                                collectLibModules(jarFile, needToDeploy);
-                            }
-                        }
-                    } else {
-                        if (!canDeployFromCustomComponentFolder(file.getName()) || platformURLMap.get(file.getName()) != null) {
-                            continue;
-                        }
-                        collectLibModules(file, needToDeploy);
-                    }
-                }
-            } catch (Exception e) {
-                ExceptionHandler.process(e);
-                continue;
-            }
-        }
-
-        // first install them locally
-        needToDeploy.forEach((k, v) -> {
-            try {
-                // install as release version if can't find mvn url from index
-                install(k, null, false, true);
-            } catch (Exception e) {
-                ExceptionHandler.process(e);
-            }
-        });
-
-        if (!deployToRemote) {
-            return;
-        }
-
-        ShareComponentsLibsJob shareJob = new ShareComponentsLibsJob(
-                Messages.getString("LocalLibraryManager.shareLibsForCustomponents"), needToDeploy, deployer);
-        shareJob.schedule();
-    }
-
-    private void collectLibModules(File jarFile, Map<File, Set<MavenArtifact>> needToDeploy) {
-        Map<String,String> mavenUris = new HashMap<String,String>();
-        guessMavenRUIFromIndex(jarFile, true, mavenUris);
-        
-        Set<MavenArtifact> artifacts = new HashSet<MavenArtifact>();
-        for(String uri: mavenUris.keySet()) {
-            MavenArtifact art = MavenUrlHelper.parseMvnUrl(uri);
-            if(art!=null) {
-                artifacts.add(art);
-            }
-        }
-        
-        needToDeploy.put(jarFile, artifacts);
-    }
-
-    private boolean canDeployFromCustomComponentFolder(String fileName) {
-        if (isSystemCacheFile(fileName) || isComponentDefinitionFileType(fileName)) {
-            return false;
-        }
-        return true;
-    }
-
     private void warnDuplicated(List<ModuleNeeded> modules, Set<String> duplicates, String type) {
         for (String lib : duplicates) {
             Set<String> components = new HashSet<>();

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/utils/DBConnectionContextUtils.java

@@ -893,6 +893,7 @@ public final class DBConnectionContextUtils {
         managerConnection.setValue(0, dbType, urlConnection, server, username, password, sidOrDatabase, port, filePath,
                 datasource, schemaOracle, additionParam, driverClassName, driverJarPath, dbVersionString);
         managerConnection.setDbRootPath(dbRootPath);
+        managerConnection.setSupportNLS(dbConn.isSupportNLS());
 
         return urlConnection;
     }
@@ -1058,6 +1059,12 @@ public final class DBConnectionContextUtils {
             cloneConn.setSQLMode(true);
         }
 
+        if(dbConn.isSetSupportNLS()) {
+            cloneConn.setSupportNLS(dbConn.isSupportNLS());
+        } else {
+            cloneConn.setSupportNLS(false);
+        }
+        
         // cloneConn.setProperties(dbConn.getProperties());
         // cloneConn.setCdcConns(dbConn.getCdcConns());
         // cloneConn.setQueries(dbConn.getQueries());

main/plugins/org.talend.metadata.managment.ui/src/main/java/org/talend/metadata/managment/ui/utils/ExtendedNodeConnectionContextUtils.java

@@ -166,7 +166,8 @@ public class ExtendedNodeConnectionContextUtils {
         KnoxUrl,
         KnoxUsername,
         KnoxPassword,
-        KnoxDirectory
+        KnoxDirectory,
+        KnoxTimeout
     }
 
     static List<IContextParameter> getContextVariables(final String prefixName, Connection conn, Set<IConnParamName> paramSet) {

main/plugins/org.talend.metadata.managment/plugin.xml

@@ -132,6 +132,22 @@
             required="true"
             uripath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.4.jar">
       </libraryNeeded>
+      <libraryNeeded
+                  context="plugin:org.talend.libraries.jdbc.oracle"
+                  language="java"
+                  message="Needed for Oracle jdbc plugin National Language Support (NLS)."
+                  mvn_uri="mvn:com.oracle.database.nls/orai18n/19.3.0.0/jar"
+                  name="orai18n-19.3.0.0.jar"
+                  required="true">
+      </libraryNeeded>
+      <libraryNeeded
+            context="plugin:org.talend.metadata.managment"
+            language="java"
+            message="Needed for plugin org.talend.metadata.managment"
+            name="hsqldb.jar" mvn_uri="mvn:org.hsqldb/hsqldb/2.7.1" 
+            required="true"
+            uripath="platform:/plugin/org.talend.libraries.jdbc.hsql/lib/hsqldb.jar">
+      </libraryNeeded>
    </extension>
    <extension
          point="org.talend.core.migrationTask">

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/AbstractTableBuilder.java

@@ -19,8 +19,6 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
 
-import metadata.managment.i18n.Messages;
-
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 import org.talend.cwm.helper.ColumnSetHelper;
@@ -29,6 +27,7 @@ import org.talend.metadata.managment.utils.MetadataConnectionUtils;
 import org.talend.utils.sql.metadata.constants.GetTable;
 import org.talend.utils.sql.metadata.constants.TableType;
 
+import metadata.managment.i18n.Messages;
 import orgomg.cwm.resource.relational.NamedColumnSet;
 
 /**
@@ -177,9 +176,9 @@ public abstract class AbstractTableBuilder<T extends NamedColumnSet> extends Cwm
         String tableComment = tablesSet.getString(GetTable.REMARKS.name());
         if (StringUtils.isBlank(tableComment)) {
             String dbProductName = getConnectionMetadata(connection).getDatabaseProductName();
-            String selectRemarkOnTable = MetadataConnectionUtils.getCommonQueryStr(dbProductName, tableName);
+            String selectRemarkOnTable = MetadataConnectionUtils.getCommonQueryStr(dbProductName);
             if (selectRemarkOnTable != null) {
-                tableComment = executeGetCommentStatement(selectRemarkOnTable);
+                tableComment = executeGetCommentStatement(selectRemarkOnTable, tableName);
             }
         }
         return tableComment;

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/CwmBuilder.java

@@ -14,14 +14,14 @@ package org.talend.core.model.metadata.builder.database;
 
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
+import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.sql.Statement;
-
-import metadata.managment.i18n.Messages;
 
 import org.apache.log4j.Logger;
 
+import metadata.managment.i18n.Messages;
+
 /**
  * @author scorreia
  *
@@ -73,14 +73,48 @@ abstract class CwmBuilder {
      */
     protected String executeGetCommentStatement(String queryStmt) {
         String comment = null;
-        Statement statement = null;
+        PreparedStatement statement = null;
         ResultSet resultSet = null;
         try {
-            statement = connection.createStatement();
-            statement.execute(queryStmt);
+            statement = connection.prepareStatement(queryStmt);
+            resultSet = statement.executeQuery();
+
+            if (resultSet != null) {
+                while (resultSet.next()) {
+                    comment = (String) resultSet.getObject(1);
+                }
+            }
+        } catch (SQLException e) {
+            // do nothing here
+        } finally {
+            // -- release resources
+            if (resultSet != null) {
+                try {
+                    resultSet.close();
+                } catch (SQLException e) {
+                    log.error(e, e);
+                }
+            }
+            if (statement != null) {
+                try {
+                    statement.close();
+                } catch (SQLException e) {
+                    log.error(e, e);
+                }
+            }
+        }
+        return comment;
+    }
+
+    protected String executeGetCommentStatement(String queryStmt, String tableName) {
+        String comment = null;
+        PreparedStatement statement = null;
+        ResultSet resultSet = null;
+        try {
+            statement = connection.prepareStatement(queryStmt);
+            statement.setString(1, tableName);
+            resultSet = statement.executeQuery();
 
-            // get the results
-            resultSet = statement.getResultSet();
             if (resultSet != null) {
                 while (resultSet.next()) {
                     comment = (String) resultSet.getObject(1);

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/ExtractMetaDataFromDataBase.java

@@ -291,6 +291,7 @@ public class ExtractMetaDataFromDataBase {
      * DOC cantoine. Method to test DataBaseConnection.
      *
      * @param dbVersionString
+     * @param supportNLS 
      *
      * @param String driverClass
      * @param String urlString pwd
@@ -299,14 +300,14 @@ public class ExtractMetaDataFromDataBase {
      * @return ConnectionStatus : the result of connection(boolean Result, String messageException)
      */
     public static ConnectionStatus testConnection(String dbType, String url, String username, String pwd, String schema,
-            final String driverClassName, final String driverJarPath, String dbVersionString, String additionalParam) {
+            final String driverClassName, final String driverJarPath, String dbVersionString, String additionalParam, boolean supportNLS) {
         return testConnection(dbType, url, username, pwd, schema, driverClassName, driverJarPath, dbVersionString,
-                additionalParam, null, null);
+                additionalParam, supportNLS, null, null);
     }
 
     public static ConnectionStatus testConnection(String dbType, String url, String username, String pwd, String schema,
             final String driverClassName, final String driverJarPath, String dbVersionString, String additionalParam,
-            StringBuffer retProposedSchema, String sidOrDatabase) {
+            boolean supportNLS, StringBuffer retProposedSchema, String sidOrDatabase) {
         Connection connection = null;
         ConnectionStatus connectionStatus = new ConnectionStatus();
         connectionStatus.setResult(false);
@@ -315,7 +316,7 @@ public class ExtractMetaDataFromDataBase {
             List list = new ArrayList();
 
             list = ExtractMetaDataUtils.getInstance().connect(dbType, url, username, pwd, driverClassName, driverJarPath,
-                    dbVersionString, additionalParam);
+                    dbVersionString, additionalParam, supportNLS);
             if (list != null && list.size() > 0) {
                 for (int i = 0; i < list.size(); i++) {
                     if (list.get(i) instanceof Connection) {
@@ -498,7 +499,7 @@ public class ExtractMetaDataFromDataBase {
         List list = metaData.getConnection(iMetadataConnection.getDbType(), url, iMetadataConnection.getUsername(),
                 iMetadataConnection.getPassword(), iMetadataConnection.getDatabase(), iMetadataConnection.getSchema(),
                 iMetadataConnection.getDriverClass(), iMetadataConnection.getDriverJarPath(),
-                iMetadataConnection.getDbVersionString(), iMetadataConnection.getAdditionalParams());
+                iMetadataConnection.getDbVersionString(), iMetadataConnection.getAdditionalParams(), iMetadataConnection.isSupportNLS());
         Connection conn = null;
         DriverShim wapperDriver = null;
 
@@ -582,7 +583,7 @@ public class ExtractMetaDataFromDataBase {
         List list = extractMeta.getConnection(iMetadataConnection.getDbType(), iMetadataConnection.getUrl(),
                 iMetadataConnection.getUsername(), iMetadataConnection.getPassword(), iMetadataConnection.getDatabase(),
                 iMetadataConnection.getSchema(), iMetadataConnection.getDriverClass(), iMetadataConnection.getDriverJarPath(),
-                iMetadataConnection.getDbVersionString(), iMetadataConnection.getAdditionalParams());
+                iMetadataConnection.getDbVersionString(), iMetadataConnection.getAdditionalParams(), iMetadataConnection.isSupportNLS());
         DriverShim wapperDriver = null;
         if (list != null && list.size() > 0) {
             for (int i = 0; i < list.size(); i++) {

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/ExtractMetaDataUtils.java

@@ -121,6 +121,8 @@ public class ExtractMetaDataUtils {
 
     private String[] ORACLE_SSL_JARS = new String[] { "oraclepki-12.2.0.1.jar", "osdt_cert-12.2.0.1.jar", //$NON-NLS-1$//$NON-NLS-2$
             "osdt_core-12.2.0.1.jar" }; //$NON-NLS-1$
+    
+    private String ORACLE_NLS_JARS = "orai18n-19.3.0.0.jar";
 
     public static final String SNOWFLAKE = "Snowflake"; //$NON-NLS-1$
 
@@ -829,6 +831,11 @@ public class ExtractMetaDataUtils {
      */
     public List getConnection(String dbType, String url, String username, String pwd, String dataBase, String schemaBase,
             final String driverClassName, final String driverJarPath, String dbVersion, String additionalParams) {
+        return getConnection(dbType, url, username, pwd, dataBase, schemaBase, driverClassName, driverJarPath, dbVersion, additionalParams, false);
+    }
+    
+    public List getConnection(String dbType, String url, String username, String pwd, String dataBase, String schemaBase,
+            final String driverClassName, final String driverJarPath, String dbVersion, String additionalParams, boolean supportNLS) {
         boolean isColsed = false;
         List conList = new ArrayList();
         try {
@@ -846,7 +853,7 @@ public class ExtractMetaDataUtils {
                 closeConnection(true); // colse before connection.
                 checkDBConnectionTimeout();
 
-                list = connect(dbType, url, username, pwd, driverClassName, driverJarPath, dbVersion, additionalParams);
+                list = connect(dbType, url, username, pwd, driverClassName, driverJarPath, dbVersion, additionalParams, supportNLS);
                 if (list != null && list.size() > 0) {
                     for (int i = 0; i < list.size(); i++) {
                         if (list.get(i) instanceof Connection) {
@@ -938,7 +945,7 @@ public class ExtractMetaDataUtils {
      * @throws Exception
      */
     public List connect(String dbType, String url, String username, String pwd, final String driverClassNameArg,
-            final String driverJarPathArg, String dbVersion, String additionalParams) throws Exception {
+            final String driverJarPathArg, String dbVersion, String additionalParams, boolean supportNLS) throws Exception {
         Connection connection = null;
         DriverShim wapperDriver = null;
         List conList = new ArrayList();
@@ -953,11 +960,18 @@ public class ExtractMetaDataUtils {
             if ((driverJarPathArg == null || driverJarPathArg.equals(""))) { //$NON-NLS-1$
                 List<String> driverNames = EDatabaseVersion4Drivers.getDrivers(dbType, dbVersion);
                 if (driverNames != null) {
+                    if(EDatabaseTypeName.ORACLEFORSID.getProduct().equals(EDatabaseTypeName.getTypeFromDbType(dbType).getProduct())) {
+                        if(supportNLS){
+                            driverNames.add(ORACLE_NLS_JARS);
+                        }
+                    }
+                    
                     if (EDatabaseTypeName.ORACLE_CUSTOM.getDisplayName().equals(dbType)
                             && StringUtils.isNotEmpty(additionalParams)) {
                         if (additionalParams.contains(SSLPreferenceConstants.TRUSTSTORE_TYPE)) {
                              driverNames.addAll(Arrays.asList(ORACLE_SSL_JARS));
                         }
+                        
                     } else if (SNOWFLAKE.equals(dbType)) { // $NON-NLS-1$
                         // TDQ-17294 msjian Support of Snowflake for DQ Datamart
                         driverNames.add(SNOWFLAKE_DRIVER_JAR);
@@ -1285,7 +1299,7 @@ public class ExtractMetaDataUtils {
         List list = getConnection(metadataConnection.getDbType(), metadataConnection.getUrl(), metadataConnection.getUsername(),
                 metadataConnection.getPassword(), metadataConnection.getDatabase(), metadataConnection.getSchema(),
                 metadataConnection.getDriverClass(), metadataConnection.getDriverJarPath(),
-                metadataConnection.getDbVersionString(), metadataConnection.getAdditionalParams());
+                metadataConnection.getDbVersionString(), metadataConnection.getAdditionalParams(), metadataConnection.isSupportNLS());
         return list;
     }
 

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/JDBCDriverLoader.java

@@ -17,6 +17,7 @@ import java.nio.charset.Charset;
 import java.security.Provider;
 import java.sql.Connection;
 import java.sql.Driver;
+import java.sql.ResultSet;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -166,6 +167,11 @@ public class JDBCDriverLoader {
                     info.put("charSet", systemCharset.displayName()); //$NON-NLS-1$
                 }
             }
+            
+            //TUP-37016:Upgrade hsqldb to 2.7.1
+            if (dbType.equals(EDatabaseTypeName.ACCESS.getXmlName()) || ConnectionUtils.isHsql(url)) {
+                System.setProperty("hsqldb.method_class_names", "net.ucanaccess.converters.*");
+            }
 
             if (additionalParams != null && !"".equals(additionalParams) && dbType.toUpperCase().contains("ORACLE")) {//$NON-NLS-1$//$NON-NLS-2$
                 if (additionalParams.contains(SSLPreferenceConstants.TRUSTSTORE_TYPE)) {
@@ -198,10 +204,20 @@ public class JDBCDriverLoader {
                 }
                 connection = wapperDriver.connect(url, info);
             }
+            
+            try {
+                ResultSet schemas = connection.getMetaData().getSchemas();
+                if(schemas.next()) {
+                    schemas.getString(1);
+                }
+            } catch (Exception e) {
+            }
+            
             // }
             // DriverManager.deregisterDriver(wapperDriver);
             // bug 9162
             list.add(connection);
+            
             list.add(wapperDriver);
             return list;
         } catch (Throwable e) {

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/manager/ExtractManager.java

@@ -470,7 +470,7 @@ public class ExtractManager {
                         metadataConnection.getUsername(), metadataConnection.getPassword(), metadataConnection.getDatabase(),
                         metadataConnection.getSchema(), metadataConnection.getDriverClass(),
                         metadataConnection.getDriverJarPath(), metadataConnection.getDbVersionString(),
-                        metadataConnection.getAdditionalParams());
+                        metadataConnection.getAdditionalParams(), metadataConnection.isSupportNLS());
                 if (list != null && list.size() > 0) {
                     for (int i = 0; i < list.size(); i++) {
                         if (list.get(i) instanceof Driver) {
@@ -574,7 +574,7 @@ public class ExtractManager {
                         metadataConnection.getUsername(), metadataConnection.getPassword(), metadataConnection.getDatabase(),
                         metadataConnection.getSchema(), metadataConnection.getDriverClass(),
                         metadataConnection.getDriverJarPath(), metadataConnection.getDbVersionString(),
-                        metadataConnection.getAdditionalParams());
+                        metadataConnection.getAdditionalParams(), metadataConnection.isSupportNLS());
                 if (list != null && list.size() > 0) {
                     for (int i = 0; i < list.size(); i++) {
                         if (list.get(i) instanceof DriverShim) {
@@ -1064,7 +1064,7 @@ public class ExtractManager {
         List connList = extractMeta.getConnection(metadataConnection.getDbType(), metadataConnection.getUrl(),
                 metadataConnection.getUsername(), metadataConnection.getPassword(), metadataConnection.getDatabase(),
                 metadataConnection.getSchema(), metadataConnection.getDriverClass(), metadataConnection.getDriverJarPath(),
-                metadataConnection.getDbVersionString(), metadataConnection.getAdditionalParams());
+                metadataConnection.getDbVersionString(), metadataConnection.getAdditionalParams(), metadataConnection.isSupportNLS());
         try {
             if (!tableInfoParameters.isUsedName()) {
                 if (tableInfoParameters.getSqlFiter() != null && !"".equals(tableInfoParameters.getSqlFiter())) { //$NON-NLS-1$

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/metadata/managment/hive/EHiveWithTezJars.java

@@ -40,7 +40,7 @@ public enum EHiveWithTezJars {
             "api-asn1-api-1.0.0-M20.jar", "api-util-1.0.0-M20.jar", "asm-3.1.jar", "avro-1.7.4.jar",
             "commons-beanutils-1.7.0.jar", "commons-beanutils-core-1.8.0.jar", "commons-compress-1.4.1.jar",
             "commons-configuration-1.6.jar", "commons-digester-1.8.jar", "commons-net-3.1.jar", "curator-client-2.6.0.jar",
-            "curator-framework-2.6.0.jar", "curator-recipes-2.6.0.jar", "gson-2.2.4.jar", "guice-3.0.jar",
+            "curator-framework-2.6.0.jar", "curator-recipes-2.6.0.jar", "gson-2.9.0.jar", "guice-3.0.jar",
             "guice-servlet-3.0.jar", "hadoop-auth-2.6.0.2.2.0.0-2041.jar", "hadoop-common-2.6.0.2.2.0.0-2041.jar",
             "hadoop-hdfs-2.6.0.2.2.0.0-2041.jar", "hadoop-yarn-api-2.6.0.2.2.0.0-2041.jar",
             "hadoop-yarn-client-2.6.0.2.2.0.0-2041.jar", "hadoop-yarn-common-2.6.0.2.2.0.0-2041.jar", "htrace-core-3.0.4.jar",
@@ -56,7 +56,7 @@ public enum EHiveWithTezJars {
             "api-asn1-api-1.0.0-M20.jar", "api-util-1.0.0-M20.jar", "asm-3.2.jar", "avro-1.7.5.jar",
             "commons-beanutils-1.7.0.jar", "commons-beanutils-core-1.8.0.jar", "commons-compress-1.4.1.jar",
             "commons-configuration-1.6.jar", "commons-digester-1.8.jar", "commons-net-3.1.jar", "curator-client-2.7.1.jar",
-            "curator-framework-2.7.1.jar", "curator-recipes-2.7.1.jar", "gson-2.2.4.jar", "guice-3.0.jar",
+            "curator-framework-2.7.1.jar", "curator-recipes-2.7.1.jar", "gson-2.9.0.jar", "guice-3.0.jar",
             "guice-servlet-3.0.jar", "hadoop-auth-2.7.1.2.3.2.0-2950.jar", "hadoop-common-2.7.1.2.3.2.0-2950.jar",
             "hadoop-hdfs-2.7.1.2.3.2.0-2950.jar", "hadoop-yarn-api-2.7.1.2.3.2.0-2950.jar",
             "hadoop-yarn-client-2.7.1.2.3.2.0-2950.jar", "hadoop-yarn-common-2.7.1.2.3.2.0-2950.jar",

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/metadata/managment/model/DBConnectionFillerImpl.java

@@ -83,6 +83,7 @@ import org.talend.metadata.managment.connection.manager.HiveConnectionManager;
 import org.talend.metadata.managment.hive.EmbeddedHiveDataBaseMetadata;
 import org.talend.metadata.managment.repository.ManagerConnection;
 import org.talend.metadata.managment.utils.DatabaseConstant;
+import org.talend.metadata.managment.utils.EDataBaseType;
 import org.talend.metadata.managment.utils.ManagementTextUtils;
 import org.talend.metadata.managment.utils.MetadataConnectionUtils;
 import org.talend.utils.sql.ConnectionUtils;
@@ -1038,8 +1039,12 @@ public class DBConnectionFillerImpl extends MetadataFillerImpl<DatabaseConnectio
                 // for CDH4 HIVE2 , the table type are MANAGED_TABLE and EXTERNAL_TABLE ......
                 // tableType = null;
             }
+            Map<String,String> tableComments = null;
+            if (!isOracle8i) {
+                tableComments = this.getTableComments(dbJDBCMetadata, catalogName, schemaPattern);
+            }
             ResultSet tables = dbJDBCMetadata.getTables(catalogName, schemaPattern, tablePattern, tableType);
-
+            boolean hasRemarksCol = hasRemarksColumn(tables);
             while (tables.next()) {
                 String coloumnName = GetTable.TABLE_SCHEM.name();
                 if (schemaPattern != null) {
@@ -1064,8 +1069,13 @@ public class DBConnectionFillerImpl extends MetadataFillerImpl<DatabaseConnectio
                 // if (!isOracle && !isOracle8i && !isOracleJdbc && tableName.startsWith("/")) { //$NON-NLS-1$
                 // continue;
                 // }
-                if (!isOracle8i) {
-                    tableComment = getTableComment(dbJDBCMetadata, tables, tableName, catalogName, schemaPattern);
+                if (hasRemarksCol) {
+                    tableComment = getRemarksFromResultSet(tables);
+                }
+                if (tableComments != null) {
+                    if (StringUtils.isEmpty(tableComment)) {
+                        tableComment = tableComments.get(tableName);
+                    }
                 }
                 MetadataTable metadatatable = null;
                 if (TableType.VIEW.toString().equals(temptableType) || ETableTypes.VIRTUAL_VIEW.getName().equals(temptableType)) {
@@ -1209,6 +1219,69 @@ public class DBConnectionFillerImpl extends MetadataFillerImpl<DatabaseConnectio
         }
         return tableComment;
     }
+    
+    private Map<String, String> getTableComments(DatabaseMetaData dbJDBCMetadata, String catalogName, String schemaPattern) {
+        Map<String, String> ret = new HashMap<String, String>();
+        PreparedStatement ps = null;
+        ResultSet rs = null;
+        try {
+            String productName = dbJDBCMetadata.getDatabaseProductName();
+            if (StringUtils.isEmpty(productName)) {
+                return ret;
+            }
+            productName = productName.replaceAll(" ", "_"); //$NON-NLS-1$ //$NON-NLS-2$
+            EDataBaseType eDataBaseType = null;
+            try {
+                eDataBaseType = EDataBaseType.valueOf(productName);
+            } catch (Exception e) {
+                eDataBaseType = EDataBaseType.Microsoft_SQL_Server;
+            }
+
+            String sqlStr = ""; //$NON-NLS-1$
+            switch (eDataBaseType) {
+            case Oracle:
+                sqlStr = "SELECT TABLE_NAME,COMMENTS FROM ALL_TAB_COMMENTS WHERE OWNER=?";
+                ps = dbJDBCMetadata.getConnection().prepareStatement(sqlStr);
+                ps.setString(1, schemaPattern.toUpperCase());
+                break;
+            case MySQL:
+                sqlStr = "SELECT TABLE_NAME,TABLE_COMMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA=?";
+                ps = dbJDBCMetadata.getConnection().prepareStatement(sqlStr);
+                ps.setString(1, catalogName);
+                break;
+            default:
+                break;
+            }
+
+            if (ps != null) {
+                rs = ps.executeQuery();
+                while (rs != null && rs.next()) {
+                    String comment = rs.getString(2);
+                    if (!StringUtils.isEmpty(comment)) {
+                        ret.put(rs.getString(1), comment);
+                    }
+                }
+            }
+        } catch (SQLException e) {
+            log.error(e, e);
+        } finally {
+            if (ps != null) {
+                try {
+                    ps.close();
+                } catch (SQLException e) {
+                    CommonExceptionHandler.process(e);
+                }
+            }
+            if (rs != null) {
+                try {
+                    rs.close();
+                } catch (SQLException e) {
+                    CommonExceptionHandler.process(e);
+                }
+            }
+        }
+        return ret;
+    }
 
     /**
      * get the Column Comment especially for oracle type.
@@ -1295,9 +1368,12 @@ public class DBConnectionFillerImpl extends MetadataFillerImpl<DatabaseConnectio
                     }
                 }
             }
-
+            Map<String, String> tableComments = null;
+            if (!flag) {
+                tableComments = this.getTableComments(dbJDBCMetadata, catalogName, schemaPattern);
+            }
             ResultSet tables = dbJDBCMetadata.getTables(catalogName, schemaPattern, tablePattern, tableType);
-
+            boolean hasRemarksCol = hasRemarksColumn(tables);
             while (tables.next()) {
                 String tableName = getStringFromResultSet(tables, GetTable.TABLE_NAME.name());
                 String temptableType = getStringFromResultSet(tables, GetTable.TABLE_TYPE.name());
@@ -1316,8 +1392,13 @@ public class DBConnectionFillerImpl extends MetadataFillerImpl<DatabaseConnectio
                 if (tableName == null || tablesToFilter.contains(tableName) || tableName.startsWith("/")) { //$NON-NLS-1$
                     continue;
                 }
-                if (!flag) {
-                    tableComment = getTableComment(dbJDBCMetadata, tables, tableName, catalogName, schemaPattern);
+                if (hasRemarksCol) {
+                    tableComment = getRemarksFromResultSet(tables);
+                }
+                if (tableComments != null) {
+                    if (StringUtils.isEmpty(tableComment)) {
+                        tableComment = tableComments.get(tableName);
+                    }
                 }
                 // create table
                 TdTable table = RelationalFactory.eINSTANCE.createTdTable();
@@ -1367,8 +1448,17 @@ public class DBConnectionFillerImpl extends MetadataFillerImpl<DatabaseConnectio
             }
         }
         try {
-
+            boolean flag = true;
+            if (pack != null) {
+                Connection c = ConnectionHelper.getConnection(pack);
+                flag = MetadataConnectionUtils.isOracle8i(c);
+            }
+            Map<String, String> tableComments = null;
+            if (!flag) {
+                tableComments = this.getTableComments(dbJDBCMetadata, catalogName, schemaPattern);
+            }
             ResultSet tables = dbJDBCMetadata.getTables(catalogName, schemaPattern, viewPattern, tableType);
+            boolean hasRemarksCol = hasRemarksColumn(tables);
             while (tables.next()) {
 
                 String tableName = getStringFromResultSet(tables, GetTable.TABLE_NAME.name());
@@ -1379,14 +1469,14 @@ public class DBConnectionFillerImpl extends MetadataFillerImpl<DatabaseConnectio
                     continue;
                 }
                 // common
-                boolean flag = true;
                 String tableComment = null;
-                if (pack != null) {
-                    Connection c = ConnectionHelper.getConnection(pack);
-                    flag = MetadataConnectionUtils.isOracle8i(c);
+                if (hasRemarksCol) {
+                    tableComment = getRemarksFromResultSet(tables);
                 }
-                if (!flag) {
-                    tableComment = getTableComment(dbJDBCMetadata, tables, tableName, catalogName, schemaPattern);
+                if (tableComments != null) {
+                    if (StringUtils.isEmpty(tableComment)) {
+                        tableComment = tableComments.get(tableName);
+                    }
                 }
                 // create table
                 TdView table = RelationalFactory.eINSTANCE.createTdView();
@@ -1425,6 +1515,24 @@ public class DBConnectionFillerImpl extends MetadataFillerImpl<DatabaseConnectio
         return valueOfString;
     }
 
+    private boolean hasRemarksColumn(ResultSet resultSet) {
+        try {
+            if (resultSet == null || resultSet.getMetaData() == null) {
+                return false;
+            }
+            int numOfCols = resultSet.getMetaData().getColumnCount();
+            for (int i = 1; i < numOfCols + 1; i++) {
+                String colName = resultSet.getMetaData().getColumnLabel(i);
+                if (StringUtils.equals(colName, GetColumn.REMARKS.name())) {
+                    return true;
+                }
+            }
+        } catch (SQLException e) {
+            CommonExceptionHandler.process(e);
+        }
+        return false;
+    }
+    
     private String getRemarksFromResultSet(ResultSet resultSet) {
         String valueOfString = null;
         try {

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/metadata/managment/repository/ManagerConnection.java

@@ -90,6 +90,8 @@ public class ManagerConnection {
     Integer id = null;
 
     String additionalParams;
+    
+    private boolean supportNLS;
 
     private String schemaOracle;
 
@@ -288,7 +290,7 @@ public class ManagerConnection {
             }
             // test the connection
             testConnection = ExtractMetaDataFromDataBase.testConnection(dbTypeString, urlConnectionString, username, password,
-                    schemaName, driverClassName, driverJarPath, dbVersionString, additionalParams, retProposedSchema,
+                    schemaName, driverClassName, driverJarPath, dbVersionString, additionalParams, supportNLS, retProposedSchema,
                     sidOrDatabase);
             isValide = testConnection.getResult();
             messageException = testConnection.getMessageException();
@@ -388,7 +390,7 @@ public class ManagerConnection {
                         metadataConnection.getUrl(), metadataConnection.getUsername(), metadataConnection.getPassword(),
                         metadataConnection.getSchema(), metadataConnection.getDriverClass(),
                         metadataConnection.getDriverJarPath(), metadataConnection.getDbVersionString(),
-                        metadataConnection.getAdditionalParams(), retProposedSchema, metadataConnection.getDatabase());
+                        metadataConnection.getAdditionalParams(), metadataConnection.isSupportNLS(), retProposedSchema, metadataConnection.getDatabase());
             }
             // qli
             // record this metadataConnection as old connection.
@@ -475,4 +477,13 @@ public class ManagerConnection {
         this.isValide = isValide;
     }
 
+    
+    /**
+     * Sets the supportNLS.
+     * @param supportNLS the supportNLS to set
+     */
+    public void setSupportNLS(boolean supportNLS) {
+        this.supportNLS = supportNLS;
+    }
+
 }

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/metadata/managment/utils/MetadataConnectionUtils.java

@@ -188,7 +188,7 @@ public class MetadataConnectionUtils {
                 }
                 list = ExtractMetaDataUtils.getInstance().connect(metadataBean.getDbType(), metadataBean.getUrl(),
                         metadataBean.getUsername(), metadataBean.getPassword(), metadataBean.getDriverClass(),
-                        metadataBean.getDriverJarPath(), metadataBean.getDbVersionString(), metadataBean.getAdditionalParams());
+                        metadataBean.getDriverJarPath(), metadataBean.getDbVersionString(), metadataBean.getAdditionalParams(), metadataBean.isSupportNLS());
             } catch (Exception e) {
                 rc.setMessage("fail to connect database!"); //$NON-NLS-1$
                 CommonExceptionHandler.process(e);
@@ -274,6 +274,7 @@ public class MetadataConnectionUtils {
             String dataBase = databaseConnection.getSID();
             String dbVersionString = databaseConnection.getDbVersionString();
             String additionalParams = databaseConnection.getAdditionalParams();
+            boolean supportNLS = databaseConnection.isSupportNLS();
 
             // MOD qiongli 2011-9-6,TDQ 3317.handle context mode
             if (databaseConnection.isContextMode()) {
@@ -313,6 +314,7 @@ public class MetadataConnectionUtils {
             metadataConnection.setUsername(userName);
             metadataConnection.setPassword(password);
             metadataConnection.setUrl(dbUrl);
+            metadataConnection.setSupportNLS(supportNLS);            
 
             // TDQ-12299: transfer the OtherParameters to metadataConnection, because create impala connection use that
             // values
@@ -803,6 +805,33 @@ public class MetadataConnectionUtils {
 
     }
 
+    public static String getCommonQueryStr(String productName) {
+        if (productName == null) {
+            return null;
+        }
+        productName = productName.replaceAll(" ", "_"); //$NON-NLS-1$ //$NON-NLS-2$
+        EDataBaseType eDataBaseType = null;
+        try {
+            eDataBaseType = EDataBaseType.valueOf(productName);
+        } catch (Exception e) {
+            eDataBaseType = EDataBaseType.Microsoft_SQL_Server;
+        }
+        String sqlStr = ""; //$NON-NLS-1$
+        switch (eDataBaseType) {
+        case Oracle:
+            sqlStr = "SELECT COMMENTS FROM USER_TAB_COMMENTS WHERE TABLE_NAME= ? "; //$NON-NLS-1$
+            break;
+        case MySQL:
+            sqlStr = "SELECT TABLE_COMMENT FROM information_schema.TABLES WHERE TABLE_NAME= ? "; //$NON-NLS-1$
+            break;
+        default:
+            sqlStr = null;
+
+        }
+        return sqlStr;
+
+    }
+
     /**
      * get Comment Query Str.
      *
@@ -1312,7 +1341,7 @@ public class MetadataConnectionUtils {
         return ExtractMetaDataUtils.getInstance().getConnection(metadataBean.getDbType(), metadataBean.getUrl(),
                 metadataBean.getUsername(), metadataBean.getPassword(), metadataBean.getDatabase(), metadataBean.getSchema(),
                 metadataBean.getDriverClass(), metadataBean.getDriverJarPath(), metadataBean.getDbVersionString(),
-                metadataBean.getAdditionalParams());
+                metadataBean.getAdditionalParams(), metadataBean.isSupportNLS());
     }
 
     /**

main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/MigrationToolService.java

@@ -23,6 +23,7 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Stream;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
@@ -240,20 +241,14 @@ public class MigrationToolService implements IMigrationToolService {
         sortMigrationTasks(toExecute);
 
         final List<MigrationTask> done = new ArrayList<MigrationTask>(storedMigrations);
-        int nbMigrationsToDo = 0;
-        for (IProjectMigrationTask task : toExecute) {
-            MigrationTask mgTask = MigrationUtil.findMigrationTask(done, task);
-            if (mgTask == null && !task.isDeprecated()) {
-                nbMigrationsToDo++;
-            }
-        }
+        boolean hasTaskToExecute = toExecute.stream()
+                .anyMatch(task -> !task.isDeprecated() && MigrationUtil.findMigrationTask(done, task) == null);
 
         // force to redo the migration task for the relations only if user ask for "clean" or if relations is empty
         // or if there is at least another migration to do.
-        if (!beforeLogon
-                && (!RelationshipItemBuilder.INDEX_VERSION.equals(project.getEmfProject().getItemsRelationVersion()) || nbMigrationsToDo > 0)) {
+        if (!beforeLogon && (!RelationshipItemBuilder.INDEX_VERSION.equals(project.getEmfProject().getItemsRelationVersion())
+                || hasTaskToExecute)) {
             // force to redo this migration task, to make sure the relationship is done correctly
-            // done.remove(RELATION_TASK);
             MigrationUtil.removeMigrationTaskById(done, RELATION_TASK);
             RelationshipItemBuilder.getInstance().unloadRelations();
 
@@ -264,41 +259,37 @@ public class MigrationToolService implements IMigrationToolService {
                     RelationshipItemBuilder.JOBLET_RELATION, true);
             // reset
             RelationshipItemBuilder.getInstance().unloadRelations();
-
-            nbMigrationsToDo++;
+            hasTaskToExecute = true;
         }
-        if (nbMigrationsToDo == 0) {
+
+        boolean checkDupContext = !beforeLogon && Boolean.getBoolean("duplicate.context.reference.check");
+        if (!hasTaskToExecute && !checkDupContext) {
             return;
         }
 
-        // force execute migration in case user copy-past items with diffrent path on the file system and refresh
-        // the studio,it may cause bug TDI-19229
-        MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.FixProjectResourceLink");
-        // force to re-generate all job poms
-        MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.GenerateJobPomMigrationTask");
-
-        if (beforeLogon) {
-            // for every migration, force reset to default maven template
-            MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.ResetMavenTemplateMigrationTask");
+        if (checkDupContext) {
+            MigrationUtil.removeMigrationTaskById(done,
+                    "org.talend.repository.model.migration.RemoveDuplicateContextReferencesMigrationTask");
         }
+        if (hasTaskToExecute) {
+            // force execute migration in case user copy-past items with diffrent path on the file system and refresh
+            // the studio,it may cause bug TDI-19229
+            MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.FixProjectResourceLink");
+            // force to re-generate all job poms
+            MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.GenerateJobPomMigrationTask");
 
-        boolean haveAnyBinFolder = false; // to avoid some problems of migration, sometimes
-        for (ERepositoryObjectType type : (ERepositoryObjectType[]) ERepositoryObjectType.values()) {
-            if (!type.hasFolder()) {
-                continue;
+            if (beforeLogon) {
+                // for every migration, force reset to default maven template
+                MigrationUtil.removeMigrationTaskById(done,
+                        "org.talend.repository.model.migration.ResetMavenTemplateMigrationTask");
             }
-            String folderName = ERepositoryObjectType.getFolderName(type);
-            if (folderName == null || "".equals(folderName)) {
-                continue;
+            boolean hasBinFolder = Stream.of((ERepositoryObjectType[]) ERepositoryObjectType.values())
+                    .filter(type -> type.hasFolder()).map(ERepositoryObjectType::getFolderName).filter(StringUtils::isNotBlank)
+                    .map(folderName -> fsProject.getFolder(folderName))
+                    .anyMatch(folder -> folder.exists() && folder.getFolder("bin").exists());
+            if (hasBinFolder) {
+                MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.RemoveBinFolderMigrationTask");
             }
-            IFolder folder = fsProject.getFolder(folderName);
-            if (folder.exists() && folder.getFolder("bin").exists()) { //$NON-NLS-1$
-                haveAnyBinFolder = true;
-                break;
-            }
-        }
-        if (haveAnyBinFolder) {
-            MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.RemoveBinFolderMigrationTask");
         }
 
         final SubProgressMonitor subProgressMonitor = new SubProgressMonitor(monitorWrap, toExecute.size());

main/plugins/org.talend.model.edit/src/main/java/org/talend/core/model/metadata/builder/connection/provider/DatabaseConnectionItemProvider.java

@@ -80,6 +80,7 @@ public class DatabaseConnectionItemProvider extends ConnectionItemProvider imple
             addCdcTypeModePropertyDescriptor(object);
             addSQLModePropertyDescriptor(object);
             addUiSchemaPropertyDescriptor(object);
+            addSupportNLSPropertyDescriptor(object);
         }
         return itemPropertyDescriptors;
     }
@@ -468,6 +469,22 @@ public class DatabaseConnectionItemProvider extends ConnectionItemProvider imple
                 false, ItemPropertyDescriptor.GENERIC_VALUE_IMAGE, null, null));
     }
 
+    /**
+     * This adds a property descriptor for the Support NLS feature.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     */
+    protected void addSupportNLSPropertyDescriptor(Object object) {
+        itemPropertyDescriptors
+                .add(createItemPropertyDescriptor(((ComposeableAdapterFactory) adapterFactory).getRootAdapterFactory(),
+                        getResourceLocator(), getString("_UI_DatabaseConnection_supportNLS_feature"),
+                        getString("_UI_PropertyDescriptor_description", "_UI_DatabaseConnection_supportNLS_feature",
+                                "_UI_DatabaseConnection_type"),
+                        ConnectionPackage.Literals.DATABASE_CONNECTION__SUPPORT_NLS, true, false, false,
+                        ItemPropertyDescriptor.BOOLEAN_VALUE_IMAGE, null, null));
+    }
+
     /**
      * This specifies how to implement {@link #getChildren} and is used to deduce an appropriate feature for an
      * {@link org.eclipse.emf.edit.command.AddCommand}, {@link org.eclipse.emf.edit.command.RemoveCommand} or
@@ -558,6 +575,7 @@ public class DatabaseConnectionItemProvider extends ConnectionItemProvider imple
         case ConnectionPackage.DATABASE_CONNECTION__CDC_TYPE_MODE:
         case ConnectionPackage.DATABASE_CONNECTION__SQL_MODE:
         case ConnectionPackage.DATABASE_CONNECTION__UI_SCHEMA:
+        case ConnectionPackage.DATABASE_CONNECTION__SUPPORT_NLS:
             fireNotifyChanged(new ViewerNotification(notification, notification.getNotifier(), false, true));
             return;
         case ConnectionPackage.DATABASE_CONNECTION__CDC_CONNS:

main/plugins/org.talend.model/model/metadata.ecore

@@ -348,6 +348,8 @@
     <eStructuralFeatures xsi:type="ecore:EAttribute" name="UiSchema" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EString"/>
     <eStructuralFeatures xsi:type="ecore:EReference" name="parameters" upperBound="-1"
         eType="#//AdditionalProperties" containment="true"/>
+    <eStructuralFeatures xsi:type="ecore:EAttribute" name="supportNLS" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EBoolean"
+        defaultValueLiteral="false" unsettable="true"/>
   </eClassifiers>
   <eClassifiers xsi:type="ecore:EClass" name="SAPConnection" eSuperTypes="#//Connection">
     <eStructuralFeatures xsi:type="ecore:EAttribute" name="Host" eType="ecore:EDataType http://www.eclipse.org/emf/2002/Ecore#//EString"/>

main/plugins/org.talend.model/model/metadata.genmodel

@@ -187,6 +187,7 @@
       <genFeatures createChild="false" ecoreFeature="ecore:EAttribute metadata.ecore#//DatabaseConnection/SQLMode"/>
       <genFeatures createChild="false" ecoreFeature="ecore:EAttribute metadata.ecore#//DatabaseConnection/UiSchema"/>
       <genFeatures property="None" children="true" createChild="true" ecoreFeature="ecore:EReference metadata.ecore#//DatabaseConnection/parameters"/>
+      <genFeatures createChild="false" ecoreFeature="ecore:EAttribute metadata.ecore#//DatabaseConnection/supportNLS"/>
     </genClasses>
     <genClasses ecoreClass="metadata.ecore#//SAPConnection">
       <genFeatures createChild="false" ecoreFeature="ecore:EAttribute metadata.ecore#//SAPConnection/Host"/>

main/plugins/org.talend.model/src/main/java/org/talend/core/model/metadata/builder/connection/ConnectionPackage.java

@@ -5238,6 +5238,15 @@ public interface ConnectionPackage extends EPackage {
      */
     int DATABASE_CONNECTION__PARAMETERS = CONNECTION_FEATURE_COUNT + 25;
 
+    /**
+     * The feature id for the '<em><b>Support NLS</b></em>' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     * @ordered
+     */
+    int DATABASE_CONNECTION__SUPPORT_NLS = CONNECTION_FEATURE_COUNT + 26;
+
     /**
      * The number of structural features of the '<em>Database Connection</em>' class.
      * <!-- begin-user-doc --> <!--
@@ -5245,7 +5254,7 @@ public interface ConnectionPackage extends EPackage {
      * @generated
      * @ordered
      */
-    int DATABASE_CONNECTION_FEATURE_COUNT = CONNECTION_FEATURE_COUNT + 26;
+    int DATABASE_CONNECTION_FEATURE_COUNT = CONNECTION_FEATURE_COUNT + 27;
 
     /**
      * The meta object id for the '{@link org.talend.core.model.metadata.builder.connection.impl.SAPConnectionImpl <em>SAP Connection</em>}' class.
@@ -21887,6 +21896,17 @@ public interface ConnectionPackage extends EPackage {
      */
     EReference getDatabaseConnection_Parameters();
 
+    /**
+     * Returns the meta object for the attribute '{@link org.talend.core.model.metadata.builder.connection.DatabaseConnection#isSupportNLS <em>Support NLS</em>}'.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @return the meta object for the attribute '<em>Support NLS</em>'.
+     * @see org.talend.core.model.metadata.builder.connection.DatabaseConnection#isSupportNLS()
+     * @see #getDatabaseConnection()
+     * @generated
+     */
+    EAttribute getDatabaseConnection_SupportNLS();
+
     /**
      * Returns the meta object for class '{@link org.talend.core.model.metadata.builder.connection.SAPConnection <em>SAP Connection</em>}'.
      * <!-- begin-user-doc --> <!-- end-user-doc -->
@@ -26439,6 +26459,14 @@ public interface ConnectionPackage extends EPackage {
          */
         EReference DATABASE_CONNECTION__PARAMETERS = eINSTANCE.getDatabaseConnection_Parameters();
 
+        /**
+         * The meta object literal for the '<em><b>Support NLS</b></em>' attribute feature.
+         * <!-- begin-user-doc -->
+         * <!-- end-user-doc -->
+         * @generated
+         */
+        EAttribute DATABASE_CONNECTION__SUPPORT_NLS = eINSTANCE.getDatabaseConnection_SupportNLS();
+
         /**
          * The meta object literal for the '{@link org.talend.core.model.metadata.builder.connection.impl.SAPConnectionImpl <em>SAP Connection</em>}' class.
          * <!-- begin-user-doc --> <!-- end-user-doc -->

main/plugins/org.talend.model/src/main/java/org/talend/core/model/metadata/builder/connection/DatabaseConnection.java

@@ -755,4 +755,54 @@ public interface DatabaseConnection extends Connection {
      */
     EMap<String, String> getParameters();
 
+    /**
+     * Returns the value of the '<em><b>Support NLS</b></em>' attribute.
+     * The default value is <code>"false"</code>.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @return the value of the '<em>Support NLS</em>' attribute.
+     * @see #isSetSupportNLS()
+     * @see #unsetSupportNLS()
+     * @see #setSupportNLS(boolean)
+     * @see org.talend.core.model.metadata.builder.connection.ConnectionPackage#getDatabaseConnection_SupportNLS()
+     * @model default="false" unsettable="true"
+     * @generated
+     */
+    boolean isSupportNLS();
+
+    /**
+     * Sets the value of the '{@link org.talend.core.model.metadata.builder.connection.DatabaseConnection#isSupportNLS <em>Support NLS</em>}' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @param value the new value of the '<em>Support NLS</em>' attribute.
+     * @see #isSetSupportNLS()
+     * @see #unsetSupportNLS()
+     * @see #isSupportNLS()
+     * @generated
+     */
+    void setSupportNLS(boolean value);
+
+    /**
+     * Unsets the value of the '{@link org.talend.core.model.metadata.builder.connection.DatabaseConnection#isSupportNLS <em>Support NLS</em>}' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @see #isSetSupportNLS()
+     * @see #isSupportNLS()
+     * @see #setSupportNLS(boolean)
+     * @generated
+     */
+    void unsetSupportNLS();
+
+    /**
+     * Returns whether the value of the '{@link org.talend.core.model.metadata.builder.connection.DatabaseConnection#isSupportNLS <em>Support NLS</em>}' attribute is set.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @return whether the value of the '<em>Support NLS</em>' attribute is set.
+     * @see #unsetSupportNLS()
+     * @see #isSupportNLS()
+     * @see #setSupportNLS(boolean)
+     * @generated
+     */
+    boolean isSetSupportNLS();
+
 } // DatabaseConnection

main/plugins/org.talend.model/src/main/java/org/talend/core/model/metadata/builder/connection/impl/ConnectionPackageImpl.java

@@ -1562,6 +1562,15 @@ public class ConnectionPackageImpl extends EPackageImpl implements ConnectionPac
         return (EReference) databaseConnectionEClass.getEStructuralFeatures().get(25);
     }
 
+    /**
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     */
+    public EAttribute getDatabaseConnection_SupportNLS() {
+        return (EAttribute) databaseConnectionEClass.getEStructuralFeatures().get(26);
+    }
+
     /**
      * <!-- begin-user-doc --> <!-- end-user-doc -->
      * @generated
@@ -4516,6 +4525,7 @@ public class ConnectionPackageImpl extends EPackageImpl implements ConnectionPac
         createEAttribute(databaseConnectionEClass, DATABASE_CONNECTION__SQL_MODE);
         createEAttribute(databaseConnectionEClass, DATABASE_CONNECTION__UI_SCHEMA);
         createEReference(databaseConnectionEClass, DATABASE_CONNECTION__PARAMETERS);
+        createEAttribute(databaseConnectionEClass, DATABASE_CONNECTION__SUPPORT_NLS);
 
         sapConnectionEClass = createEClass(SAP_CONNECTION);
         createEAttribute(sapConnectionEClass, SAP_CONNECTION__HOST);
@@ -5294,6 +5304,9 @@ public class ConnectionPackageImpl extends EPackageImpl implements ConnectionPac
         initEReference(getDatabaseConnection_Parameters(), this.getAdditionalProperties(), null, "parameters", null, 0, -1,
                 DatabaseConnection.class, !IS_TRANSIENT, !IS_VOLATILE, IS_CHANGEABLE, IS_COMPOSITE, IS_RESOLVE_PROXIES,
                 !IS_UNSETTABLE, IS_UNIQUE, !IS_DERIVED, IS_ORDERED);
+        initEAttribute(getDatabaseConnection_SupportNLS(), ecorePackage.getEBoolean(), "supportNLS", "false", 0, 1,
+                DatabaseConnection.class, !IS_TRANSIENT, !IS_VOLATILE, IS_CHANGEABLE, IS_UNSETTABLE, !IS_ID, IS_UNIQUE,
+                !IS_DERIVED, IS_ORDERED);
 
         initEClass(sapConnectionEClass, SAPConnection.class, "SAPConnection", !IS_ABSTRACT, !IS_INTERFACE,
                 IS_GENERATED_INSTANCE_CLASS);

main/plugins/org.talend.model/src/main/java/org/talend/core/model/metadata/builder/connection/impl/DatabaseConnectionImpl.java

@@ -51,6 +51,7 @@ import org.talend.core.model.metadata.builder.connection.DatabaseConnection;
  *   <li>{@link org.talend.core.model.metadata.builder.connection.impl.DatabaseConnectionImpl#isSQLMode <em>SQL Mode</em>}</li>
  *   <li>{@link org.talend.core.model.metadata.builder.connection.impl.DatabaseConnectionImpl#getUiSchema <em>Ui Schema</em>}</li>
  *   <li>{@link org.talend.core.model.metadata.builder.connection.impl.DatabaseConnectionImpl#getParameters <em>Parameters</em>}</li>
+ *   <li>{@link org.talend.core.model.metadata.builder.connection.impl.DatabaseConnectionImpl#isSupportNLS <em>Support NLS</em>}</li>
  * </ul>
  *
  * @generated
@@ -563,6 +564,35 @@ public class DatabaseConnectionImpl extends ConnectionImpl implements DatabaseCo
      */
     protected EMap<String, String> parameters;
 
+    /**
+     * The default value of the '{@link #isSupportNLS() <em>Support NLS</em>}' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @see #isSupportNLS()
+     * @generated
+     * @ordered
+     */
+    protected static final boolean SUPPORT_NLS_EDEFAULT = false;
+
+    /**
+     * The cached value of the '{@link #isSupportNLS() <em>Support NLS</em>}' attribute.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @see #isSupportNLS()
+     * @generated
+     * @ordered
+     */
+    protected boolean supportNLS = SUPPORT_NLS_EDEFAULT;
+
+    /**
+     * This is true if the Support NLS attribute has been set.
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     * @ordered
+     */
+    protected boolean supportNLSESet = true;
+
     /**
      * <!-- begin-user-doc --> <!-- end-user-doc -->
      * @generated
@@ -1173,6 +1203,54 @@ public class DatabaseConnectionImpl extends ConnectionImpl implements DatabaseCo
         return parameters;
     }
 
+    /**
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     */
+    public boolean isSupportNLS() {
+        return supportNLS;
+    }
+
+    /**
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     */
+    public void setSupportNLS(boolean newSupportNLS) {
+        boolean oldSupportNLS = supportNLS;
+        supportNLS = newSupportNLS;
+        boolean oldSupportNLSESet = supportNLSESet;
+        supportNLSESet = true;
+        if (eNotificationRequired())
+            eNotify(new ENotificationImpl(this, Notification.SET, ConnectionPackage.DATABASE_CONNECTION__SUPPORT_NLS,
+                    oldSupportNLS, supportNLS, !oldSupportNLSESet));
+    }
+
+    /**
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     */
+    public void unsetSupportNLS() {
+        boolean oldSupportNLS = supportNLS;
+        boolean oldSupportNLSESet = supportNLSESet;
+        supportNLS = SUPPORT_NLS_EDEFAULT;
+        supportNLSESet = false;
+        if (eNotificationRequired())
+            eNotify(new ENotificationImpl(this, Notification.UNSET, ConnectionPackage.DATABASE_CONNECTION__SUPPORT_NLS,
+                    oldSupportNLS, SUPPORT_NLS_EDEFAULT, oldSupportNLSESet));
+    }
+
+    /**
+     * <!-- begin-user-doc -->
+     * <!-- end-user-doc -->
+     * @generated
+     */
+    public boolean isSetSupportNLS() {
+        return supportNLSESet;
+    }
+
     /**
      * <!-- begin-user-doc --> <!-- end-user-doc -->
      * @generated
@@ -1267,6 +1345,8 @@ public class DatabaseConnectionImpl extends ConnectionImpl implements DatabaseCo
                 return getParameters();
             else
                 return getParameters().map();
+        case ConnectionPackage.DATABASE_CONNECTION__SUPPORT_NLS:
+            return isSupportNLS();
         }
         return super.eGet(featureID, resolve, coreType);
     }
@@ -1335,6 +1415,8 @@ public class DatabaseConnectionImpl extends ConnectionImpl implements DatabaseCo
                 return getParameters();
             else
                 return getParameters().map();
+        case ConnectionPackage.DATABASE_CONNECTION__SUPPORT_NLS:
+            return isSupportNLS();
         }
         return super.eGet(featureID, resolve, coreType);
     }
@@ -1424,6 +1506,9 @@ public class DatabaseConnectionImpl extends ConnectionImpl implements DatabaseCo
         case ConnectionPackage.DATABASE_CONNECTION__PARAMETERS:
             ((EStructuralFeature.Setting) getParameters()).set(newValue);
             return;
+        case ConnectionPackage.DATABASE_CONNECTION__SUPPORT_NLS:
+            setSupportNLS((Boolean) newValue);
+            return;
         }
         super.eSet(featureID, newValue);
     }
@@ -1513,6 +1598,9 @@ public class DatabaseConnectionImpl extends ConnectionImpl implements DatabaseCo
         case ConnectionPackage.DATABASE_CONNECTION__PARAMETERS:
             getParameters().clear();
             return;
+        case ConnectionPackage.DATABASE_CONNECTION__SUPPORT_NLS:
+            unsetSupportNLS();
+            return;
         }
         super.eUnset(featureID);
     }
@@ -1578,6 +1666,8 @@ public class DatabaseConnectionImpl extends ConnectionImpl implements DatabaseCo
             return UI_SCHEMA_EDEFAULT == null ? uiSchema != null : !UI_SCHEMA_EDEFAULT.equals(uiSchema);
         case ConnectionPackage.DATABASE_CONNECTION__PARAMETERS:
             return parameters != null && !parameters.isEmpty();
+        case ConnectionPackage.DATABASE_CONNECTION__SUPPORT_NLS:
+            return isSetSupportNLS();
         }
         return super.eIsSet(featureID);
     }
@@ -1643,6 +1733,11 @@ public class DatabaseConnectionImpl extends ConnectionImpl implements DatabaseCo
             result.append("<unset>");
         result.append(", UiSchema: ");
         result.append(uiSchema);
+        result.append(", supportNLS: ");
+        if (supportNLSESet)
+            result.append(supportNLS);
+        else
+            result.append("<unset>");
         result.append(')');
         return result.toString();
     }

main/plugins/org.talend.repository.items.importexport.ui/src/main/java/org/talend/repository/items/importexport/ui/wizard/imports/ImportItemsWizardPage.java

@@ -721,6 +721,10 @@ public class ImportItemsWizardPage extends WizardPage {
 
         ImportDependencyRelationsHelper.getInstance().checkImportRelationDependency(checkedNodeList, toSelectSet,
                 nodesBuilder.getAllImportItemNode());
+
+
+        // to make doCheckStateChanged execute from ContainerCheckedTreeViewer.setCheckedElements(Object[])
+        filteredCheckboxTree.getViewer().setCheckedElements(new Object[0]);
         filteredCheckboxTree.getViewer().setCheckedElements(toSelectSet.toArray());
     }
 

main/plugins/org.talend.repository.items.importexport/src/main/java/org/talend/repository/items/importexport/handlers/imports/ImportBasicHandler.java

@@ -23,6 +23,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -51,6 +52,8 @@ import org.eclipse.emf.ecore.impl.EObjectImpl;
 import org.eclipse.emf.ecore.resource.Resource;
 import org.eclipse.emf.ecore.resource.ResourceSet;
 import org.eclipse.emf.ecore.util.EcoreUtil;
+import org.eclipse.emf.ecore.xmi.XMLResource;
+import org.eclipse.emf.ecore.xmi.impl.XMLParserPoolImpl;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.runtime.model.repository.ERepositoryStatus;
@@ -1121,6 +1124,13 @@ public class ImportBasicHandler extends AbstractImportExecutableHandler {
             }
             stream = manager.getStream(itemPath, importItem);
             Resource resource = createResource(importItem, itemPath, byteArray);
+            //TUP-36820:Add options for improving performance for deserialization (loading) of large XML resource
+            Map optionMap = new HashMap();
+            optionMap.put(XMLResource.OPTION_DEFER_ATTACHMENT, Boolean.TRUE);
+            optionMap.put(XMLResource.OPTION_DEFER_IDREF_RESOLUTION, Boolean.TRUE);
+            optionMap.put(XMLResource.OPTION_USE_PARSER_POOL, new XMLParserPoolImpl());
+            optionMap.put(XMLResource.OPTION_USE_XML_NAME_TO_FEATURE_MAP, new HashMap());
+            optionMap.put(XMLResource.OPTION_USE_DEPRECATED_METHODS, Boolean.FALSE);
 
             if (byteArray) {
                 // TDI-24612
@@ -1133,16 +1143,16 @@ public class ImportBasicHandler extends AbstractImportExecutableHandler {
                     baos.write(buf, 0, i);
                 }
                 ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
-                resource.load(bais, null);
+                resource.load(bais, optionMap);
             } else {
-                resource.load(stream, null);
+                resource.load(stream, optionMap);
             }
 
             for (ReferenceFileItem rfItem : (List<ReferenceFileItem>) item.getReferenceResources()) {
                 itemPath = getReferenceItemPath(importItem.getPath(), rfItem);
                 stream = manager.getStream(itemPath, importItem);
                 Resource rfResource = createResource(importItem, itemPath, true);
-                rfResource.load(stream, null);
+                rfResource.load(stream, optionMap);
             }
 
             Iterator<EObject> itRef = item.eCrossReferences().iterator();

main/plugins/org.talend.repository.items.importexport/src/main/java/org/talend/repository/items/importexport/handlers/imports/ImportDependencyRelationsHelper.java

@@ -132,11 +132,12 @@ public class ImportDependencyRelationsHelper {
                     id = split[1];
                 }
             }
+            boolean isGlobalRoutine = RelationshipItemBuilder.ROUTINE_RELATION.equals(relation.getType());
             if (RelationshipItemBuilder.LATEST_VERSION.equals(relation.getVersion())) {
-                relatedNode = getLatestVersionItemImportNode(id, projectLabel, allImportItemNodesList);
+                relatedNode = getLatestVersionItemImportNode(id, projectLabel, allImportItemNodesList, isGlobalRoutine);
             } else {
-                relatedNode = getItemImportNodeByIdVersion(id, projectLabel, relation.getVersion(),
-                        allImportItemNodesList);
+                relatedNode = getItemImportNodeByIdVersion(id, relation.getVersion(), projectLabel,
+                        allImportItemNodesList, isGlobalRoutine);
             }
             if (relatedNode != null && !toSelectSet.contains(relatedNode)) {
                 // avoid loop
@@ -148,13 +149,14 @@ public class ImportDependencyRelationsHelper {
     }
 
     public ItemImportNode getLatestVersionItemImportNode(String id, String projectTecLabel,
-            List<ItemImportNode> allImportItemNodesList) {
+            List<ItemImportNode> allImportItemNodesList, boolean isGlobalRoutine) {
         List<ItemImportNode> allItemImportNodesById = getItemImportNode(allImportItemNodesList, node -> {
+            Property property = node.getItemRecord().getProperty();
             boolean projectFlag = true;
             if (StringUtils.isNotBlank(projectTecLabel)) {
                 projectFlag = node.getProjectNode().getProject().getTechnicalLabel().equals(projectTecLabel);
             }
-            return node.getItemRecord().getProperty().getId().equals(id) && projectFlag;
+            return (isGlobalRoutine ? property.getLabel().equals(id) : property.getId().equals(id)) && projectFlag;
         });
         Optional<ItemImportNode> optional = allItemImportNodesById.stream().max((node1, node2) -> VersionUtils
                 .compareTo(node1.getItemRecord().getProperty().getVersion(), node2.getItemRecord().getProperty().getVersion()));
@@ -162,14 +164,15 @@ public class ImportDependencyRelationsHelper {
     }
 
     public ItemImportNode getItemImportNodeByIdVersion(String id, String version, String projectTecLabel,
-            List<ItemImportNode> allImportItemNodesList) {
+            List<ItemImportNode> allImportItemNodesList, boolean isGlobalRoutine) {
         List<ItemImportNode> importNodeList = getItemImportNode(allImportItemNodesList, node -> {
             boolean projectFlag = true;
             if (StringUtils.isNotBlank(projectTecLabel)) {
                 projectFlag = node.getProjectNode().getProject().getTechnicalLabel().equals(projectTecLabel);
             }
             Property property = node.getItemRecord().getProperty();
-            return property.getId().equals(id) && property.getVersion().equals(version) && projectFlag;
+            return (isGlobalRoutine ? property.getLabel().equals(id) : property.getId().equals(id))
+                    && property.getVersion().equals(version) && projectFlag;
         });
         return importNodeList == null || importNodeList.isEmpty() ? null : importNodeList.get(0);
     }

main/plugins/org.talend.repository.metadata/src/main/java/org/talend/repository/metadata/i18n/messages.properties

@@ -1008,6 +1008,8 @@ DatabaseForm.hc.link.repository=Repository
 DatabaseForm.hc.link.title=Hadoop Cluster
 DatabaseForm.helpInfo.installDriverLink.url=https://document-link.us.cloud.talend.com/ts_ig_install-external-modules?version=73&lang=en&env=prd
 DatabaseForm.helpInfo.installDriverLink.label=How to install a driver
+DatabaseForm.supportnls=Support NLS
+DatabaseForm.supportnls.warntip=This setting takes effect only after restarting the Studio.
 DatabaseTableFilterForm.allSynonyms=All synonyms
 DatabaseTableFilterForm.edit=Edit...
 DatabaseTableFilterForm.editFilterName=Edit Filter Name

main/plugins/org.talend.repository.metadata/src/main/java/org/talend/repository/metadata/ui/actions/metadata/AbstractCreateTableAction.java

@@ -17,11 +17,6 @@ import java.util.Collection;
 import java.util.List;
 
 import org.apache.log4j.Logger;
-import org.eclipse.core.runtime.IProgressMonitor;
-import org.eclipse.core.runtime.IStatus;
-import org.eclipse.core.runtime.Status;
-import org.eclipse.core.runtime.jobs.IJobChangeEvent;
-import org.eclipse.core.runtime.jobs.JobChangeAdapter;
 import org.eclipse.emf.common.util.EList;
 import org.eclipse.emf.ecore.util.EcoreUtil;
 import org.eclipse.jface.dialogs.MessageDialog;
@@ -29,7 +24,6 @@ import org.eclipse.jface.wizard.IWizard;
 import org.eclipse.jface.wizard.WizardDialog;
 import org.eclipse.swt.widgets.Display;
 import org.eclipse.ui.PlatformUI;
-import org.eclipse.ui.progress.UIJob;
 import org.talend.commons.exception.LoginException;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.core.GlobalServiceRegister;
@@ -830,177 +824,145 @@ public abstract class AbstractCreateTableAction extends AbstractCreateAction {
 
     private void openDatabaseTableWizard(final DatabaseConnectionItem item, final MetadataTable metadataTable,
             final boolean forceReadOnly, final RepositoryNode node, final boolean creation) {
-        UIJob job = new UIJob(Messages.getString("CreateTableAction.action.createTitle")) { //$NON-NLS-1$
+        String name = "User action : " + getText(); //$NON-NLS-1$
+        RepositoryWorkUnit<Object> repositoryWorkUnit = new RepositoryWorkUnit<Object>(name, this) {
 
             @Override
-            public IStatus runInUIThread(final IProgressMonitor monitor) {
-                String name = "User action : " + getText(); //$NON-NLS-1$
-                RepositoryWorkUnit<Object> repositoryWorkUnit = new RepositoryWorkUnit<Object>(name, this) {
-
-                    @Override
-                    protected void run() throws LoginException, PersistenceException {
-
-                        monitor.beginTask(Messages.getString("CreateTableAction.action.createTitle"), IProgressMonitor.UNKNOWN); //$NON-NLS-1$
-
-                        if (!monitor.isCanceled()) {
-                            final ManagerConnection managerConnection = new ManagerConnection();
-
-                            DatabaseConnection connection = (DatabaseConnection) item.getConnection();
-                            boolean useKrb = Boolean.valueOf(connection.getParameters().get(
-                                    ConnParameterKeys.CONN_PARA_KEY_USE_KRB));
-                            // TUP-596 : Update the context name in connection when the user does a context switch in DI
-                            String oldContextName = connection.getContextName();
-                            IMetadataConnection metadataConnection = ConvertionHelper.convert(connection, false, null);
-                            String newContextName = connection.getContextName();
-                            if (oldContextName != null && newContextName != null && !oldContextName.equals(newContextName)) {
-                                if (node != null && node.getObject() != null && node.getObject().getProperty() != null) {
-                                    Item itemTemp = node.getObject().getProperty().getItem();
-                                    if (itemTemp != null && itemTemp instanceof ConnectionItem) {
-                                        ConnectionItem connItem = (ConnectionItem) itemTemp;
-                                        SwitchContextGroupNameImpl.getInstance().updateContextGroup(connItem, newContextName);
-                                    }
-                                }
-                            }
-                            boolean isTcomDB = false;
-                            IGenericDBService dbService = null;
-                            if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericDBService.class)) {
-                                dbService = (IGenericDBService) GlobalServiceRegister.getDefault().getService(
-                                        IGenericDBService.class);
-                            }
-                            if(dbService != null){
-                                for(ERepositoryObjectType type : dbService.getExtraTypes()){
-                                    if(type.getLabel().equals(metadataConnection.getDbType())){
-                                        isTcomDB = true;
-                                    }
-                                }
-                            }
-                            if (!metadataConnection.getDbType().equals(EDatabaseConnTemplate.GODBC.getDBDisplayName())
-                                    && !metadataConnection.getDbType().equals(EDatabaseConnTemplate.ACCESS.getDBDisplayName())
-                                    && !metadataConnection.getDbType().equals(
-                                            EDatabaseConnTemplate.GENERAL_JDBC.getDBDisplayName())
-                                    && !isTcomDB) {
-                                // TODO 1. To identify if it is hive connection.
-                                String hiveMode = (String) metadataConnection
-                                        .getParameter(ConnParameterKeys.CONN_PARA_KEY_HIVE_MODE);
-                                if (EDatabaseTypeName.HIVE.getDisplayName().equals(metadataConnection.getDbType())) {
-                                    // metadataConnection.setDriverJarPath((String)metadataConnection
-                                    // .getParameter(ConnParameterKeys.CONN_PARA_KEY_METASTORE_CONN_DRIVER_JAR));
-                                    if (HiveModeInfo.get(hiveMode) == HiveModeInfo.EMBEDDED) {
-                                        JavaSqlFactory.doHivePreSetup((DatabaseConnection) metadataConnection
-                                                .getCurrentConnection());
-                                    }
-                                } else if (EDatabaseTypeName.IMPALA.getDisplayName().equals(metadataConnection.getDbType())) {
-                                    DatabaseConnection originalValueConnection = null;
-                                    IRepositoryContextService repositoryContextService = CoreRuntimePlugin.getInstance()
-                                            .getRepositoryContextService();
-                                    if (repositoryContextService != null) {
-                                        originalValueConnection = repositoryContextService
-                                                .cloneOriginalValueConnection(connection, false, null);
-                                    }
-                                    if (originalValueConnection != null) {
-                                        metadataConnection.setUrl(originalValueConnection.getURL());
-                                    }
-                                } else {
-                                    String genUrl = DatabaseConnStrUtil.getURLString(metadataConnection.getDbType(),
-                                            metadataConnection.getDbVersionString(), metadataConnection.getServerName(),
-                                            metadataConnection.getUsername(), metadataConnection.getPassword(),
-                                            metadataConnection.getPort(), metadataConnection.getDatabase(),
-                                            metadataConnection.getFileFieldName(), metadataConnection.getDataSourceName(),
-                                            metadataConnection.getDbRootPath(), metadataConnection.getAdditionalParams());
-                                    if (!(metadataConnection.getDbType().equals(EDatabaseConnTemplate.IMPALA.getDBDisplayName()) && useKrb)) {
-                                        metadataConnection.setUrl(genUrl);
-                                    }
-                                }
-
-                            }
-                            // bug 23508:even open type is metaTable,not connection,we always need the connection's
-                            // datapackage to find the table schema when click the retrieve schema button
-                            if (connection != null) {
-                                EList<orgomg.cwm.objectmodel.core.Package> dp = connection.getDataPackage();
-                                Collection<Package> newDataPackage = EcoreUtil.copyAll(dp);
-                                ConnectionHelper.addPackages(newDataPackage,
-                                        (DatabaseConnection) metadataConnection.getCurrentConnection());
-                            }
-                            if (creation) {
-                                String hiveMode = (String) metadataConnection
-                                        .getParameter(ConnParameterKeys.CONN_PARA_KEY_HIVE_MODE);
-                                if (EDatabaseTypeName.HIVE.getDisplayName().equals(metadataConnection.getDbType())) {
-                                    try {
-                                        HiveConnectionManager.getInstance().checkConnection(metadataConnection);
-                                    } catch (Exception e) {
-                                        e.printStackTrace();
-                                    }
-                                } else {
-                                    managerConnection.check(metadataConnection);
-                                }
-
-                                // ExtractMetaDataUtils.metadataCon = metadataConnection;
-                                // when open,set use synonyms false.
-                                ExtractMetaDataUtils.getInstance().setUseAllSynonyms(false);
-
-                                IProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
-                                boolean repositoryObjectEditable = factory.isEditableAndLockIfPossible(node.getObject());
-                                if (!repositoryObjectEditable) {
-                                    boolean flag = MessageDialog.openConfirm(PlatformUI.getWorkbench().getActiveWorkbenchWindow()
-                                            .getShell(), Messages.getString("CreateTableAction.action.Warning"),
-                                            Messages.getString("CreateTableAction.action.NotLockMessage"));
-                                    if (flag) {
-                                        DatabaseTableWizard databaseTableWizard = new DatabaseTableWizard(
-                                                PlatformUI.getWorkbench(), creation, node.getObject(), metadataTable,
-                                                getExistingNames(), forceReadOnly, managerConnection, metadataConnection);
-
-                                        WizardDialog wizardDialog = new WizardDialog(PlatformUI.getWorkbench()
-                                                .getActiveWorkbenchWindow().getShell(), databaseTableWizard);
-                                        wizardDialog.setBlockOnOpen(true);
-                                        handleWizard(node, wizardDialog);
-                                    }
-                                } else {
-                                    DatabaseTableWizard databaseTableWizard = new DatabaseTableWizard(PlatformUI.getWorkbench(),
-                                            creation, node.getObject(), metadataTable, getExistingNames(), forceReadOnly,
-                                            managerConnection, metadataConnection);
-
-                                    WizardDialog wizardDialog = new WizardDialog(PlatformUI.getWorkbench()
-                                            .getActiveWorkbenchWindow().getShell(), databaseTableWizard);
-                                    wizardDialog.setBlockOnOpen(true);
-                                    handleWizard(node, wizardDialog);
-                                }
-                            } else {
-                                // added for bug 16595
-                                // no need connect to database when double click one schema.
-                                final boolean skipStep = true;
-
-                                DatabaseTableWizard databaseTableWizard = new DatabaseTableWizard(PlatformUI.getWorkbench(),
-                                        creation, node.getObject(), metadataTable, getExistingNames(), forceReadOnly,
-                                        managerConnection, metadataConnection);
-                                databaseTableWizard.setSkipStep(skipStep);
-                                WizardDialog wizardDialog = new WizardDialog(PlatformUI.getWorkbench().getActiveWorkbenchWindow()
-                                        .getShell(), databaseTableWizard);
-                                handleWizard(node, wizardDialog);
-                            }
+            protected void run() throws LoginException, PersistenceException {
+                final ManagerConnection managerConnection = new ManagerConnection();
 
+                DatabaseConnection connection = (DatabaseConnection) item.getConnection();
+                boolean useKrb = Boolean.valueOf(connection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_USE_KRB));
+                // TUP-596 : Update the context name in connection when the user does a context switch in DI
+                String oldContextName = connection.getContextName();
+                IMetadataConnection metadataConnection = ConvertionHelper.convert(connection, false, null);
+                String newContextName = connection.getContextName();
+                if (oldContextName != null && newContextName != null && !oldContextName.equals(newContextName)) {
+                    if (node != null && node.getObject() != null && node.getObject().getProperty() != null) {
+                        Item itemTemp = node.getObject().getProperty().getItem();
+                        if (itemTemp != null && itemTemp instanceof ConnectionItem) {
+                            ConnectionItem connItem = (ConnectionItem) itemTemp;
+                            SwitchContextGroupNameImpl.getInstance().updateContextGroup(connItem, newContextName);
+                        }
+                    }
+                }
+                boolean isTcomDB = false;
+                IGenericDBService dbService = null;
+                if (GlobalServiceRegister.getDefault().isServiceRegistered(IGenericDBService.class)) {
+                    dbService = (IGenericDBService) GlobalServiceRegister.getDefault().getService(IGenericDBService.class);
+                }
+                if (dbService != null) {
+                    for (ERepositoryObjectType type : dbService.getExtraTypes()) {
+                        if (type.getLabel().equals(metadataConnection.getDbType())) {
+                            isTcomDB = true;
+                        }
+                    }
+                }
+                if (!metadataConnection.getDbType().equals(EDatabaseConnTemplate.GODBC.getDBDisplayName())
+                        && !metadataConnection.getDbType().equals(EDatabaseConnTemplate.ACCESS.getDBDisplayName())
+                        && !metadataConnection.getDbType().equals(EDatabaseConnTemplate.GENERAL_JDBC.getDBDisplayName())
+                        && !isTcomDB) {
+                    // TODO 1. To identify if it is hive connection.
+                    String hiveMode = (String) metadataConnection.getParameter(ConnParameterKeys.CONN_PARA_KEY_HIVE_MODE);
+                    if (EDatabaseTypeName.HIVE.getDisplayName().equals(metadataConnection.getDbType())) {
+                        // metadataConnection.setDriverJarPath((String)metadataConnection
+                        // .getParameter(ConnParameterKeys.CONN_PARA_KEY_METASTORE_CONN_DRIVER_JAR));
+                        if (HiveModeInfo.get(hiveMode) == HiveModeInfo.EMBEDDED) {
+                            JavaSqlFactory.doHivePreSetup((DatabaseConnection) metadataConnection.getCurrentConnection());
+                        }
+                    } else if (EDatabaseTypeName.IMPALA.getDisplayName().equals(metadataConnection.getDbType())) {
+                        DatabaseConnection originalValueConnection = null;
+                        IRepositoryContextService repositoryContextService = CoreRuntimePlugin.getInstance()
+                                .getRepositoryContextService();
+                        if (repositoryContextService != null) {
+                            originalValueConnection = repositoryContextService.cloneOriginalValueConnection(connection, false,
+                                    null);
+                        }
+                        if (originalValueConnection != null) {
+                            metadataConnection.setUrl(originalValueConnection.getURL());
+                        }
+                    } else {
+                        String genUrl = DatabaseConnStrUtil.getURLString(metadataConnection.getDbType(),
+                                metadataConnection.getDbVersionString(), metadataConnection.getServerName(),
+                                metadataConnection.getUsername(), metadataConnection.getPassword(), metadataConnection.getPort(),
+                                metadataConnection.getDatabase(), metadataConnection.getFileFieldName(),
+                                metadataConnection.getDataSourceName(), metadataConnection.getDbRootPath(),
+                                metadataConnection.getAdditionalParams());
+                        if (!(metadataConnection.getDbType().equals(EDatabaseConnTemplate.IMPALA.getDBDisplayName()) && useKrb)) {
+                            metadataConnection.setUrl(genUrl);
                         }
                     }
-                };
-                repositoryWorkUnit.setAvoidUnloadResources(isAvoidUnloadResources());
-                IRepositoryService repositoryService = (IRepositoryService) GlobalServiceRegister.getDefault().getService(
-                        IRepositoryService.class);
-                repositoryService.getProxyRepositoryFactory().executeRepositoryWorkUnit(repositoryWorkUnit);
-                monitor.done();
-                return Status.OK_STATUS;
-            };
-        };
-        job.setUser(true);
-        job.addJobChangeListener(new JobChangeAdapter() {
 
-            @Override
-            public void done(IJobChangeEvent event) {
-                if (!event.getResult().isOK()) {
-                    log.error(event.getResult().getMessage(), event.getResult().getException());
-                } // else eveything is fine so do not log anything
+                }
+                // bug 23508:even open type is metaTable,not connection,we always need the connection's
+                // datapackage to find the table schema when click the retrieve schema button
+                if (connection != null) {
+                    EList<orgomg.cwm.objectmodel.core.Package> dp = connection.getDataPackage();
+                    Collection<Package> newDataPackage = EcoreUtil.copyAll(dp);
+                    ConnectionHelper.addPackages(newDataPackage, (DatabaseConnection) metadataConnection.getCurrentConnection());
+                }
+                if (creation) {
+                    String hiveMode = (String) metadataConnection.getParameter(ConnParameterKeys.CONN_PARA_KEY_HIVE_MODE);
+                    if (EDatabaseTypeName.HIVE.getDisplayName().equals(metadataConnection.getDbType())) {
+                        try {
+                            HiveConnectionManager.getInstance().checkConnection(metadataConnection);
+                        } catch (Exception e) {
+                            e.printStackTrace();
+                        }
+                    } else {
+                        managerConnection.check(metadataConnection);
+                    }
+
+                    // ExtractMetaDataUtils.metadataCon = metadataConnection;
+                    // when open,set use synonyms false.
+                    ExtractMetaDataUtils.getInstance().setUseAllSynonyms(false);
+
+                    IProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
+                    boolean repositoryObjectEditable = factory.isEditableAndLockIfPossible(node.getObject());
+                    if (!repositoryObjectEditable) {
+                        boolean flag = MessageDialog.openConfirm(PlatformUI.getWorkbench().getActiveWorkbenchWindow().getShell(),
+                                Messages.getString("CreateTableAction.action.Warning"),
+                                Messages.getString("CreateTableAction.action.NotLockMessage"));
+                        if (flag) {
+                            DatabaseTableWizard databaseTableWizard = new DatabaseTableWizard(PlatformUI.getWorkbench(), creation,
+                                    node.getObject(), metadataTable, getExistingNames(), forceReadOnly, managerConnection,
+                                    metadataConnection);
+
+                            WizardDialog wizardDialog = new WizardDialog(
+                                    PlatformUI.getWorkbench().getActiveWorkbenchWindow().getShell(), databaseTableWizard);
+                            wizardDialog.setBlockOnOpen(true);
+                            handleWizard(node, wizardDialog);
+                        }
+                    } else {
+                        DatabaseTableWizard databaseTableWizard = new DatabaseTableWizard(PlatformUI.getWorkbench(), creation,
+                                node.getObject(), metadataTable, getExistingNames(), forceReadOnly, managerConnection,
+                                metadataConnection);
+
+                        WizardDialog wizardDialog = new WizardDialog(
+                                PlatformUI.getWorkbench().getActiveWorkbenchWindow().getShell(), databaseTableWizard);
+                        wizardDialog.setBlockOnOpen(true);
+                        handleWizard(node, wizardDialog);
+                    }
+                } else {
+                    // added for bug 16595
+                    // no need connect to database when double click one schema.
+                    final boolean skipStep = true;
+
+                    DatabaseTableWizard databaseTableWizard = new DatabaseTableWizard(PlatformUI.getWorkbench(), creation,
+                            node.getObject(), metadataTable, getExistingNames(), forceReadOnly, managerConnection,
+                            metadataConnection);
+                    databaseTableWizard.setSkipStep(skipStep);
+                    WizardDialog wizardDialog = new WizardDialog(PlatformUI.getWorkbench().getActiveWorkbenchWindow().getShell(),
+                            databaseTableWizard);
+                    handleWizard(node, wizardDialog);
+                }
+
             }
-        });
-        job.schedule();
-
+        };
+        repositoryWorkUnit.setAvoidUnloadResources(isAvoidUnloadResources());
+        IRepositoryService repositoryService = (IRepositoryService) GlobalServiceRegister.getDefault()
+                .getService(IRepositoryService.class);
+        repositoryService.getProxyRepositoryFactory().executeRepositoryWorkUnit(repositoryWorkUnit);
     }
 
     /**

main/plugins/org.talend.repository.metadata/src/main/java/org/talend/repository/ui/wizards/metadata/connection/database/DatabaseForm.java

@@ -66,6 +66,7 @@ import org.eclipse.swt.widgets.Button;
 import org.eclipse.swt.widgets.Composite;
 import org.eclipse.swt.widgets.Event;
 import org.eclipse.swt.widgets.Group;
+import org.eclipse.swt.widgets.Label;
 import org.eclipse.swt.widgets.Link;
 import org.eclipse.swt.widgets.Listener;
 import org.eclipse.swt.widgets.Text;
@@ -567,6 +568,8 @@ public class DatabaseForm extends AbstractForm {
 
     private LabelledFileField dataprocPathToCredentialsForHiveTxt;
 
+    private Button isOracleSupportNLS;
+
     /**
      * Constructor to use by a Wizard to create a new database connection.
      *
@@ -684,6 +687,8 @@ public class DatabaseForm extends AbstractForm {
         if (getConnection().getDbVersionString() != null) {
             dbVersionCombo.setText(getConnection().getDbVersionString());
         }
+        
+        isOracleSupportNLS.setSelection(getConnection().isSupportNLS());
 
         fileField.setText(getConnection().getFileFieldName());
         directoryField.setText(getConnection().getDBRootPath());
@@ -859,6 +864,7 @@ public class DatabaseForm extends AbstractForm {
         dbVersionCombo.setReadOnly(isReadOnly());
         datasourceText.setReadOnly(isReadOnly());
         additionParamText.setReadOnly(isReadOnly());
+        isOracleSupportNLS.setEnabled(!isReadOnly());
         fileField.setReadOnly(isReadOnly());
         mappingFileText.setReadOnly(isReadOnly());
         mappingSelectButton.setEnabled(isReadOnly());
@@ -1035,6 +1041,7 @@ public class DatabaseForm extends AbstractForm {
         additionParamText = new LabelledText(typeDbCompositeParent, Messages.getString("DatabaseForm.AddParams"), 2); //$NON-NLS-1$
         additionalJDBCSettingsText = new LabelledText(typeDbCompositeParent,
                 Messages.getString("DatabaseForm.hive.additionalJDBCSettings"), 2); //$NON-NLS-1$
+        createOracleUIForNLS(typeDbCompositeParent);
 
         String[] extensions = { "*.*" }; //$NON-NLS-1$
         fileField = new LabelledFileField(typeDbCompositeParent, Messages.getString("DatabaseForm.mdbFile"), extensions); //$NON-NLS-1$
@@ -1062,6 +1069,38 @@ public class DatabaseForm extends AbstractForm {
         createHivePropertiesFields(typeDbCompositeParent);
     }
 
+    private void createOracleUIForNLS(Composite parent) {
+        supportNLSContainer = new Composite(parent, SWT.NONE);
+        GridData containerLayoutData = new GridData();
+        containerLayoutData.exclude = true;
+        supportNLSContainer.setLayoutData(containerLayoutData);
+        supportNLSContainer.setVisible(false);
+        
+        GridLayout containerLayout = new GridLayout();
+        containerLayout.numColumns = 2;
+        containerLayout.marginWidth = 0;
+        containerLayout.marginHeight = 0;
+        containerLayout.horizontalSpacing = 0;
+        containerLayout.makeColumnsEqualWidth = false;
+        supportNLSContainer.setLayout(containerLayout);
+        
+        isOracleSupportNLS = new Button(supportNLSContainer, SWT.CHECK);
+        isOracleSupportNLS.setText(Messages.getString("DatabaseForm.supportnls"));//$NON-NLS-1$
+        GridData oracleSupportNLSLayoutData = new GridData();
+        isOracleSupportNLS.setLayoutData(oracleSupportNLSLayoutData);
+        isOracleSupportNLS.addSelectionListener(new SelectionAdapter() {
+
+            @Override
+            public void widgetSelected(SelectionEvent e) {
+                getConnection().setSupportNLS(isOracleSupportNLS.getSelection());
+            }
+        });
+        imageLabel = new Label(supportNLSContainer, SWT.NONE);
+        imageLabel.setImage(ImageProvider.getImage(EImage.WARNING_ICON));
+        imageLabel.setLayoutData(new GridData());
+        imageLabel.setToolTipText(Messages.getString("DatabaseForm.supportnls.warntip"));
+    }
+    
     private void createHiveDataprocField(Composite parent) {
         dataprocProjectIdForHiveTxt = new LabelledText(parent, Messages.getString("DatabaseForm.dataproc.projectId"), 2);//$NON-NLS-1$
         dataprocClusterIdForHiveTxt = new LabelledText(parent, Messages.getString("DatabaseForm.dataproc.clusterId"), 2); //$NON-NLS-1$
@@ -4464,6 +4503,7 @@ public class DatabaseForm extends AbstractForm {
                     enableDbVersion() ? versionStr : null, metadataconnection.getOtherParameters());
 
             managerConnection.setDbRootPath(directoryField.getText());
+            managerConnection.setSupportNLS(isOracleSupportNLS.getSelection());
 
         }
         IPreferenceStore store = CoreUIPlugin.getDefault().getPreferenceStore();
@@ -5071,6 +5111,10 @@ public class DatabaseForm extends AbstractForm {
 
                         getConnection().setDbVersionString(version.getVersionValue());
 
+                        boolean supportNLSOracleVersion = oracleVersionEnable()
+                                && isSupportNLSOracleVersion(dbVersionCombo.getText());
+                        showOracleSupportNLS(supportNLSOracleVersion, supportNLSOracleVersion);
+                        
                     }
                     urlConnectionStringText.setText(getStringConnection());
                     checkFieldsValue();
@@ -6508,6 +6552,10 @@ public class DatabaseForm extends AbstractForm {
 
     private static String DEFAULT_HIVE_METASTORE_PORT = "9083";
 
+    private Label imageLabel;
+
+    private Composite supportNLSContainer;
+
     /**
      * SetEditable fields.
      *
@@ -6665,6 +6713,7 @@ public class DatabaseForm extends AbstractForm {
             showIfHiveMetastore();
             showIfSupportEncryption();
             showIfAuthentication();
+            showOracleSupportNLS(oracleVersionEnable() && isSupportNLSOracleVersion(dbVersionCombo.getText()), visible);
             hideHiveExecutionFields(!doSupportTez());
 
             urlConnectionStringText.setEditable(!visible);
@@ -6969,6 +7018,34 @@ public class DatabaseForm extends AbstractForm {
         compositeGroupDbSettings.layout();
     }
 
+    private void showOracleSupportNLS(boolean show, boolean editable) {
+        GridData layoutData = (GridData) supportNLSContainer.getLayoutData();
+        layoutData.exclude = !show;
+        supportNLSContainer.setLayoutData(layoutData);
+        supportNLSContainer.setVisible(show);
+        imageLabel.setVisible(show);
+        if(!show) {
+            isOracleSupportNLS.setSelection(false);
+            getConnection().setSupportNLS(false);
+        } else {
+            isOracleSupportNLS.setEnabled(editable);
+        }
+        supportNLSContainer.getParent().layout();
+    }
+    
+    private boolean isSupportNLSOracleVersion(String dbVersionString) {
+        if (!EDatabaseVersion4Drivers.ORACLE_8.getVersionDisplay().equals(dbVersionString) 
+                && !EDatabaseVersion4Drivers.ORACLE_9.getVersionDisplay().equals(dbVersionString) 
+                && !EDatabaseVersion4Drivers.ORACLE_10.getVersionDisplay().equals(dbVersionString)
+                && !EDatabaseVersion4Drivers.ORACLE_11.getVersionDisplay().equals(dbVersionString)
+                && !EDatabaseVersion4Drivers.ORACLE_12.getVersionDisplay().equals(dbVersionString)
+                ) {
+            return true;
+        }
+        
+        return false;
+    }
+
     private void collectContextParams() {
         collectHiveContextParams();
         collectHBaseContextParams();
@@ -7365,6 +7442,7 @@ public class DatabaseForm extends AbstractForm {
 
         jDBCschemaText.setEditable(!isContextMode());
 
+        isOracleSupportNLS.setEnabled(!isContextMode());
         generalMappingFileText.setEditable(!isContextMode());
         mappingFileText.setEditable(!isContextMode());
         if (isContextMode()) {

main/plugins/org.talend.repository.metadata/src/main/java/org/talend/repository/ui/wizards/metadata/table/database/DefaultSelectorTreeViewerProvider.java

@@ -0,0 +1,35 @@
+// ============================================================================
+//
+// Copyright (C) 2006-2021 Talend Inc. - www.talend.com
+//
+// This source code is available under agreement available at
+// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
+//
+// You should have received a copy of the agreement
+// along with this program; if not, write to Talend SA
+// 9 rue Pages 92150 Suresnes, France
+//
+// ============================================================================
+package org.talend.repository.ui.wizards.metadata.table.database;
+
+import java.util.List;
+
+import org.talend.core.model.metadata.builder.database.TableNode;
+
+/**
+ * wzhang class global comment. Detailled comment
+ */
+public class DefaultSelectorTreeViewerProvider extends SelectorTreeViewerProvider {
+
+    public DefaultSelectorTreeViewerProvider() {
+        super();
+    }
+
+    @Override
+    public Object[] getChildren(Object parentElement) {
+        TableNode tableNode = (TableNode) parentElement;
+        List<TableNode> child = tableNode.getChildren();
+        return child.toArray();
+    }
+
+}

main/plugins/org.talend.repository.metadata/src/main/java/org/talend/repository/ui/wizards/metadata/table/database/SelectorTableForm.java

@@ -16,6 +16,8 @@ import java.lang.reflect.InvocationTargetException;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.Driver;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.text.Collator;
 import java.util.ArrayList;
@@ -80,6 +82,7 @@ import org.talend.commons.ui.swt.formtools.Form;
 import org.talend.commons.ui.swt.formtools.UtilsButton;
 import org.talend.commons.utils.data.text.IndiceHelper;
 import org.talend.commons.utils.threading.TalendCustomThreadPoolExecutor;
+import org.talend.core.database.EDatabase4DriverClassName;
 import org.talend.core.database.EDatabaseTypeName;
 import org.talend.core.database.conn.ConnParameterKeys;
 import org.talend.core.model.metadata.IMetadataConnection;
@@ -89,6 +92,7 @@ import org.talend.core.model.metadata.MetadataToolHelper;
 import org.talend.core.model.metadata.builder.connection.DatabaseConnection;
 import org.talend.core.model.metadata.builder.connection.MetadataColumn;
 import org.talend.core.model.metadata.builder.connection.MetadataTable;
+import org.talend.core.model.metadata.builder.database.DriverShim;
 import org.talend.core.model.metadata.builder.database.ExtractMetaDataFromDataBase;
 import org.talend.core.model.metadata.builder.database.ExtractMetaDataFromDataBase.ETableTypes;
 import org.talend.core.model.metadata.builder.database.ExtractMetaDataUtils;
@@ -388,6 +392,7 @@ public class SelectorTableForm extends AbstractForm {
                 // TDI-28768 after add checkStateProvider, the catalog and schema checked status become true , then
                 // force them to false
                 if (parentNode.getType() == TableNode.CATALOG || parentNode.getType() == TableNode.SCHEMA) {
+                    retrieveSchema(parentNode);
                     needUpdate = false;
                 }
                 boolean firstExpand = false;
@@ -396,6 +401,9 @@ public class SelectorTableForm extends AbstractForm {
                 } else {
                     firstExpand = true;
                 }
+                if (treeItem.isDisposed()) {
+                    return;
+                }
                 for (TreeItem item : treeItem.getItems()) {
                     if (item.getData() != null) {
                         TableNode node = (TableNode) item.getData();
@@ -466,7 +474,7 @@ public class SelectorTableForm extends AbstractForm {
             viewProvider = provider.getMetadataViewProvider();
 
         } else {
-            viewProvider = new SelectorTreeViewerProvider();
+            viewProvider = new DefaultSelectorTreeViewerProvider();
         }
 
         viewer.setLabelProvider(viewProvider);
@@ -476,7 +484,237 @@ public class SelectorTableForm extends AbstractForm {
         scrolledCompositeFileViewer.setContent(tree);
         scrolledCompositeFileViewer.setMinSize(tree.computeSize(SWT.DEFAULT, SWT.DEFAULT));
     }
+    
+    private void retrieveSchema(TableNode tableNode) {
+        
+        IRunnableWithProgress runnable = new IRunnableWithProgress() {
 
+            @Override
+            public void run(IProgressMonitor monitor) throws InvocationTargetException, InterruptedException {
+                monitor.beginTask(Messages.getString("CreateTableAction.action.createTitle"), IProgressMonitor.UNKNOWN); //$NON-NLS-1$
+                
+                List<TableNode> child = tableNode.getChildren();
+                boolean extended = false;
+                if (!child.isEmpty()) {
+                    for (TableNode node : child) {
+                        if (node.getType() == TableNode.TABLE) {
+                            extended = true;
+                            break;
+                        }
+                    }
+                }
+                // if extended is true, means table already got,no need to get again.
+                if (extended) {
+                    return;
+                }
+                
+                List<MetadataTable> tableList = new ArrayList<MetadataTable>();
+                retrieveAllSubNodes(tableNode, tableList);
+
+                Display.getDefault().syncExec(() -> {
+                    viewer.setInput(tableNodeList);
+                    viewer.expandToLevel(tableNode, viewer.ALL_LEVELS);
+                });
+                monitor.done();
+            }
+
+        };
+
+        try {
+            this.parentWizardPage.getWizard().getContainer().run(true, true, runnable);
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        }
+    }
+
+    private void retrieveAllSubNodes(TableNode tableNode, List<MetadataTable> tableList) {
+        tableList.clear();
+        List<TableNode> child = tableNode.getChildren();
+        boolean extended = false;
+        if (!child.isEmpty()) {
+            for (TableNode node : child) {
+                if (node.getType() == TableNode.TABLE) {
+                    extended = true;
+                    break;
+                }
+            }
+        }
+        // if extended is true, means table already got,no need to get again.
+        if (extended) {
+            return;
+        }
+
+        IMetadataConnection metadataConn = tableNode.getMetadataConn();
+
+        Connection conn = null;
+        Driver driver = null;
+
+        DatabaseMetaData dbMetaData = null;
+        ExtractMetaDataUtils extractMeta = ExtractMetaDataUtils.getInstance();
+        // Added by Marvin Wang on Mar. 13, 2013 for loading hive jars dynamically, refer to TDI-25072.
+        if (EDatabaseTypeName.HIVE.getXmlName().equalsIgnoreCase(metadataConn.getDbType())) {
+            try {
+                dbMetaData = HiveConnectionManager.getInstance().extractDatabaseMetaData(metadataConn);
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+        } else if (EDatabaseTypeName.IMPALA.getDisplayName().equalsIgnoreCase(metadataConn.getDbType())) {
+            try {
+                dbMetaData = ImpalaConnectionManager.getInstance().createConnection(metadataConn).getMetaData();
+            } catch (Exception e) {
+                ExceptionHandler.process(e);
+            }
+        } else {
+            List list = extractMeta.getConnectionList(metadataConn);
+            if (list != null && !list.isEmpty()) {
+                for (int i = 0; i < list.size(); i++) {
+                    if (list.get(i) instanceof Connection) {
+                        conn = (Connection) list.get(i);
+                    }
+                    if (list.get(i) instanceof DriverShim) {
+                        driver = (DriverShim) list.get(i);
+                    }
+                }
+            }
+            dbMetaData = extractMeta.getDatabaseMetaData(conn, metadataConn.getDbType(), metadataConn.isSqlMode(),
+                    metadataConn.getDatabase());
+        }
+
+        int type = tableNode.getType();
+        orgomg.cwm.objectmodel.core.Package pack = null;
+
+        if (type == tableNode.CATALOG) {
+            if (tableNode.getChildren().isEmpty()) {
+                pack = tableNode.getCatalog();
+            } else {
+                for (TableNode n : tableNode.getChildren()) {
+                    retrieveAllSubNodes(n, tableList);
+                }
+            }
+        } else if (type == tableNode.SCHEMA) {
+            pack = tableNode.getSchema();
+        }
+        try {
+            if (pack != null) {
+                TableInfoParameters paras = tableNode.getParas();
+                List<ETableTypes> paraType = paras.getTypes();
+                Set<String> availableTableTypes = new HashSet<String>();
+                for (ETableTypes tableType : paraType) {
+                    availableTableTypes.add(tableType.getName());
+                }
+                // get all tables/views depending the filter selected
+
+                Set<String> tableNameFilter = null;
+
+                if (!paras.isUsedName()) {
+                    tableNameFilter = new HashSet<String>();
+                    if (paras.getSqlFiter() != null && !"".equals(paras.getSqlFiter())) { //$NON-NLS-1$
+                        PreparedStatement stmt = extractMeta.getConn().prepareStatement(paras.getSqlFiter());
+                        extractMeta.setQueryStatementTimeout(stmt);
+                        ResultSet rsTables = stmt.executeQuery();
+                        while (rsTables.next()) {
+                            String nameKey = rsTables.getString(1).trim();
+                            tableNameFilter.add(nameKey);
+                        }
+                        rsTables.close();
+                        stmt.close();
+                    }
+                } else {
+                    tableNameFilter = paras.getNameFilters();
+                }
+
+                List<MetadataTable> tempListTables = new ArrayList<MetadataTable>();
+                MetadataFillFactory dbInstance = MetadataFillFactory.getDBInstance(metadataConn);
+                for (String filter : tableNameFilter) {
+                    tempListTables = dbInstance.fillAll(pack, dbMetaData, metadataConn, null, filter,
+                            availableTableTypes.toArray(new String[] {}));
+                    for (MetadataTable table : tempListTables) {
+                        boolean contains = false;
+                        for (MetadataTable inListTable : tableList) {
+                            if (inListTable.getName().equals(table.getName())) {
+                                contains = true;
+                                break;
+                            }
+                        }
+                        if (!contains) {
+                            tableList.add(table);
+                        }
+                    }
+                }
+                if (tableNameFilter.isEmpty()) {
+                    tempListTables = dbInstance.fillAll(pack, dbMetaData, metadataConn, null, null,
+                            availableTableTypes.toArray(new String[] {}));
+                    for (MetadataTable table : tempListTables) {
+                        boolean contains = false;
+                        for (MetadataTable inListTable : tableList) {
+                            if (inListTable.getName().equals(table.getName())) {
+                                contains = true;
+                                break;
+                            }
+                        }
+                        if (!contains) {
+                            tableList.add(table);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            ExceptionHandler.process(e);
+        } finally {
+            String dbType = metadataConn.getDbType();
+            // bug 22619
+            String driverClass = metadataConn.getDriverClass();
+            if (conn != null) {
+                ConnectionUtils.closeConnection(conn);
+            }
+            // for specific db such as derby
+            if (driver != null) {
+                if ((driverClass != null && driverClass.equals(EDatabase4DriverClassName.JAVADB_EMBEDED.getDriverClass()))
+                        || (dbType != null && (dbType.equals(EDatabaseTypeName.JAVADB_EMBEDED.getDisplayName())
+                                || dbType.equals(EDatabaseTypeName.JAVADB_DERBYCLIENT.getDisplayName())
+                                || dbType.equals(EDatabaseTypeName.JAVADB_JCCJDBC.getDisplayName())
+                                || dbType.equals(EDatabaseTypeName.HSQLDB_IN_PROGRESS.getDisplayName())))) {
+                    try {
+                        driver.connect("jdbc:derby:;shutdown=true", null); //$NON-NLS-1$
+                    } catch (SQLException e) {
+                        // exception of shutdown success. no need to catch.
+                    }
+                }
+            }
+        }
+
+        if (!(tableNode.getType() == TableNode.CATALOG && pack == null)) {
+            transferToTableNode(tableList, tableNode);
+        }
+
+    }
+
+    protected void transferToTableNode(List<MetadataTable> list, TableNode parentNode) {
+        if (list != null && !list.isEmpty()) {
+            for (MetadataTable table : list) {
+                if (table instanceof TdTable) {
+                    TdTable td = (TdTable) table;
+                    TableNode tableNode = new TableNode();
+                    tableNode.setType(TableNode.TABLE);
+                    tableNode.setValue(td.getLabel());
+                    tableNode.setItemType(td.getTableType());
+                    tableNode.setTable(td);
+                    tableNode.setParent(parentNode);
+                    parentNode.addChild(tableNode);
+                } else if (table instanceof TdView) {
+                    TdView tv = (TdView) table;
+                    TableNode tableNode = new TableNode();
+                    tableNode.setType(TableNode.TABLE);
+                    tableNode.setValue(tv.getLabel());
+                    tableNode.setItemType(tv.getTableType());
+                    tableNode.setView(tv);
+                    tableNode.setParent(parentNode);
+                    parentNode.addChild(tableNode);
+                }
+            }
+        }
+    }
+    
     private final Collator col = Collator.getInstance(Locale.getDefault());
 
     /**
@@ -943,6 +1181,10 @@ public class SelectorTableForm extends AbstractForm {
                         }
                         if (canAdd) {
                             tableNodes.add(schemaNode);
+                            if (schemaNode.getValue() != null && StringUtils.isEmpty(schemaNode.getValue().trim())) {
+                                List<MetadataTable> tableList = new ArrayList<MetadataTable>();
+                                retrieveAllSubNodes(schemaNode, tableList);
+                            }
                         }
                     }
                 }

main/plugins/org.talend.repository.view.example/components/tExampleComponent/tExampleComponent_java.xml

@@ -201,8 +201,8 @@ SHOW_IF="USE_FILE_AMBIGUOUS=='true'">
 	<CODEGENERATION>
 		<IMPORTS>
 			<IMPORT NAME="Talen File Enhanced"
-				MODULE="talend_file_enhanced-1.1.jar"
-				MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" REQUIRED="true" />
+				MODULE="talend_file_enhanced-1.3.jar"
+				MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.3" REQUIRED="true" />
 			<IMPORT NAME="Talend_CSV" MODULE="talendcsv.jar"
 				MVN="mvn:org.talend.libraries/talendcsv/6.0.0"
 				REQUIRED="true" />

main/plugins/org.talend.utils/pom_minimal.xml

@@ -60,7 +60,7 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-text</artifactId>
-            <version>1.1</version>
+            <version>1.10.0</version>
         </dependency>
 		<dependency>
   			<groupId>org.slf4j</groupId>

main/plugins/org.talend.utils/pom_server.xml

@@ -55,7 +55,7 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-text</artifactId>
-            <version>1.1</version>
+            <version>1.10.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>

test/plugins/org.talend.librariesmanager.test/pom.xml

@@ -22,12 +22,12 @@
        <version>1.0.0</version>
        <type>dll</type>
      </dependency>
-    <dependency>
+    <!--dependency>
        <groupId>org.talend.studio.test</groupId>
        <artifactId>nexus.upload.test.new</artifactId>
        <version>1.0.0</version>
        <type>exe</type>
-     </dependency>
+     </dependency-->
     <dependency>
        <groupId>org.talend.studio.test</groupId>
        <artifactId>nexus.upload.test.new</artifactId>
@@ -45,12 +45,12 @@
        <version>1.0.0</version>
        <type>dll</type>
      </dependency>
-    <dependency>
+    <!--dependency>
        <groupId>org.talend.studio.test</groupId>
        <artifactId>nexus.upload.test.old</artifactId>
        <version>1.0.0</version>
        <type>exe</type>
-     </dependency>
+     </dependency-->
     <dependency>
        <groupId>org.talend.studio.test</groupId>
        <artifactId>nexus.upload.test.old</artifactId>

test/plugins/org.talend.librariesmanager.test/src/org/talend/librariesmanager/nexus/Nexus3RepositoryHandlerTest.java

@@ -66,7 +66,7 @@ public class Nexus3RepositoryHandlerTest {
     private static Properties nexusprops = new Properties();
     private static ArtifactRepositoryBean customNexusServer;
     private static IRepositoryArtifactHandler repHandler;
-    private static String[]  types = new String[] {"jar", "pom", "exe", "zip", "dll"};
+    private static String[]  types = new String[] {"jar", "pom", "zip", "dll"};
     
     @BeforeClass
     public static void init() throws FileNotFoundException, IOException  {

test/plugins/org.talend.metadata.managment.test/src/org/talend/core/model/metadata/builder/database/manager/dbs/AbstractTest4ExtractManager.java

@@ -36,10 +36,6 @@ import java.util.Properties;
 import java.util.Set;
 
 import org.eclipse.emf.common.util.BasicEList;
-import org.eclipse.emf.common.util.URI;
-import org.eclipse.emf.ecore.resource.Resource;
-import org.eclipse.emf.ecore.resource.ResourceSet;
-import org.eclipse.emf.ecore.resource.impl.ResourceSetImpl;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Ignore;
@@ -60,6 +56,7 @@ import org.talend.core.model.metadata.builder.database.manager.ExtractManagerFac
 import org.talend.cwm.relational.RelationalFactory;
 import org.talend.cwm.relational.TdColumn;
 import org.talend.cwm.relational.TdTable;
+
 import orgomg.cwm.objectmodel.core.Feature;
 
 /**
@@ -650,7 +647,7 @@ public class AbstractTest4ExtractManager {
 
         List list = extractMeta.getConnection(metadataConn.getDbType(), metadataConn.getUrl(), metadataConn.getUsername(),
                 metadataConn.getPassword(), metadataConn.getDatabase(), metadataConn.getSchema(), metadataConn.getDriverClass(),
-                metadataConn.getDriverJarPath(), metadataConn.getDbVersionString(), metadataConn.getAdditionalParams());
+                metadataConn.getDriverJarPath(), metadataConn.getDbVersionString(), metadataConn.getAdditionalParams(), metadataConn.isSupportNLS());
         assertTrue(list.size() == 0);
         list.add(conn);
         assertTrue(list.size() != 0);

test/plugins/org.talend.repository.items.importexport.test/src/org/talend/repository/items/importexport/handlers/imports/ImportDependencyRelationsHelperTest.java

@@ -16,6 +16,7 @@ import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import org.eclipse.core.resources.IProject;
@@ -41,6 +42,7 @@ import org.talend.core.model.properties.ItemRelations;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.PropertiesFactory;
 import org.talend.core.model.properties.Property;
+import org.talend.core.model.properties.RoutineItem;
 import org.talend.core.model.properties.User;
 import org.talend.core.model.properties.impl.PropertiesFactoryImpl;
 import org.talend.core.model.relationship.Relation;
@@ -93,8 +95,10 @@ public class ImportDependencyRelationsHelperTest {
             allImportItemNodesList.add(importNode);
             propertyList.add(property);
         }
+        // test0 --> test1 --> test2
         createRelations(propertyList);
         Property property2 = propertyList.get(2);
+        // label test2 version 0.7
         Property property3 = PropertiesFactory.eINSTANCE.createProperty();
         property3.setId(property2.getId());
         property3.setLabel(property2.getLabel());
@@ -108,6 +112,34 @@ public class ImportDependencyRelationsHelperTest {
         projectNode.addChild(importNode);
         allImportItemNodesList.add(importNode);
         propertyList.add(property3);
+
+        Property routineProperty = PropertiesFactory.eINSTANCE.createProperty();
+        routineProperty.setId(ProxyRepositoryFactory.getInstance().getNextId());
+        routineProperty.setLabel("testRoutine");
+        routineProperty.setVersion("0.1");
+        RoutineItem routineItem = PropertiesFactory.eINSTANCE.createRoutineItem();
+        routineProperty.setItem(routineItem);
+        ImportItem routineItemRecord = new ImportItem(new Path(fakePath + "/" + technicalLabel + "/code/routines"
+                + routineProperty.getLabel() + "_" + routineProperty.getVersion() + ".item"));
+        routineItemRecord.setProperty(routineProperty);
+        ItemImportNode routineImportNode = new ItemImportNode(routineItemRecord);
+        projectNode.addChild(routineImportNode);
+        allImportItemNodesList.add(routineImportNode);
+        propertyList.add(routineProperty);
+        Property routineProperty1 = PropertiesFactory.eINSTANCE.createProperty();
+        routineProperty1.setId(routineProperty.getId());
+        routineProperty1.setLabel("testRoutine");
+        routineProperty1.setVersion("0.7");
+        RoutineItem routineItem1 = PropertiesFactory.eINSTANCE.createRoutineItem();
+        routineProperty1.setItem(routineItem1);
+        ImportItem routineItemRecord1 = new ImportItem(new Path(fakePath + "/" + technicalLabel + "/code/routines"
+                + routineProperty1.getLabel() + "_" + routineProperty1.getVersion() + ".item"));
+        routineItemRecord1.setProperty(routineProperty1);
+        ItemImportNode routineImportNode1 = new ItemImportNode(routineItemRecord1);
+        projectNode.addChild(routineImportNode1);
+        allImportItemNodesList.add(routineImportNode1);
+        propertyList.add(routineProperty1);
+
         ImportCacheHelper.getInstance().getPathWithProjects().put(fakeProjectPath, project.getEmfProject());
     }
 
@@ -131,7 +163,35 @@ public class ImportDependencyRelationsHelperTest {
         helperInstance.checkImportRelationDependency(checkedNodeList, toSelectSet, allImportItemNodesList);
         Assert.assertTrue(toSelectSet.size() == 3);
 
-        // to test loop dependency
+        Map<Relation, Set<Relation>> importItemsRelations = helperInstance.getImportItemsRelations(fakeProjectPath);
+        Property jobProperty = propertyList.get(3);
+        Relation baseRelation = new Relation();
+        baseRelation.setId(jobProperty.getId());
+        baseRelation.setType(RelationshipItemBuilder.JOB_RELATION);
+        baseRelation.setVersion(jobProperty.getVersion());
+        Relation relatedRelation = new Relation();
+        relatedRelation.setId(propertyList.get(4).getLabel());
+        relatedRelation.setType(RelationshipItemBuilder.ROUTINE_RELATION);
+        relatedRelation.setVersion(RelationshipItemBuilder.LATEST_VERSION);
+        Set<Relation> relationSet = new HashSet<Relation>();
+        relationSet.add(relatedRelation);
+        importItemsRelations.put(baseRelation, relationSet);
+        toSelectSet.clear();
+        toSelectSet.add(allImportItemNodesList.get(0));
+        helperInstance.checkImportRelationDependency(checkedNodeList, toSelectSet, allImportItemNodesList);
+        Assert.assertTrue(toSelectSet.size() == 4);
+
+    }
+
+    @Test
+    public void checkImportRelationWithLoopDependency() {
+        helperInstance.clear();
+        helperInstance.loadRelations(fakeProjectPath, project.getEmfProject().getItemsRelations());
+        Set<ItemImportNode> toSelectSet = new HashSet<ItemImportNode>();
+        List<ItemImportNode> checkedNodeList = new ArrayList<ItemImportNode>();
+        checkedNodeList.add(allImportItemNodesList.get(0));
+        toSelectSet.add(allImportItemNodesList.get(0));
+        // to test loop dependency test0 --> test1 --> test2 --> test0
         Map<Relation, Set<Relation>> importItemsRelations = helperInstance.getImportItemsRelations(fakeProjectPath);
         Property property3 = propertyList.get(3);
         Relation baseRelation = new Relation();
@@ -145,20 +205,87 @@ public class ImportDependencyRelationsHelperTest {
         Set<Relation> relationSet = new HashSet<Relation>();
         relationSet.add(relatedRelation);
         importItemsRelations.put(baseRelation, relationSet);
-        toSelectSet.clear();
-        toSelectSet.add(allImportItemNodesList.get(0));
         helperInstance.checkImportRelationDependency(checkedNodeList, toSelectSet, allImportItemNodesList);
         Assert.assertTrue(toSelectSet.size() == 3);
     }
 
+    @Test
+    public void checkImportRelationDependencyWithMultiVersion() {
+        helperInstance.clear();
+        helperInstance.loadRelations(fakeProjectPath, project.getEmfProject().getItemsRelations());
+        Set<ItemImportNode> toSelectSet = new HashSet<ItemImportNode>();
+        List<ItemImportNode> checkedNodeList = new ArrayList<ItemImportNode>();
+
+        // test1 --> test2 latest (0.7)
+        // test1 --> test2 0.1
+        Relation relatedRelation = new Relation();
+        relatedRelation.setId(propertyList.get(3).getId());
+        relatedRelation.setType(RelationshipItemBuilder.JOB_RELATION);
+        relatedRelation.setVersion("0.1");
+
+        Map<Relation, Set<Relation>> importItemsRelations = helperInstance.getImportItemsRelations(fakeProjectPath);
+        String test1_id = propertyList.get(1).getId();
+        Optional<Relation> optional = importItemsRelations.keySet().stream().filter(relation -> relation.getId().equals(test1_id))
+                .findFirst();
+        Assert.assertTrue(optional.isPresent());
+        importItemsRelations.get(optional.get()).add(relatedRelation);
+        checkedNodeList.add(allImportItemNodesList.get(1));
+        toSelectSet.add(allImportItemNodesList.get(1));
+        helperInstance.checkImportRelationDependency(checkedNodeList, toSelectSet, allImportItemNodesList);
+        Assert.assertTrue(toSelectSet.size() == 3);
+
+        toSelectSet.clear();
+        checkedNodeList.add(allImportItemNodesList.get(0));
+        toSelectSet.add(allImportItemNodesList.get(0));
+        helperInstance.checkImportRelationDependency(checkedNodeList, toSelectSet, allImportItemNodesList);
+        Assert.assertTrue(toSelectSet.size() == 4);
+    }
+
+    @Test
+    public void testGetItemImportNodeByIdVersion() {
+        ItemImportNode theVersionNode = helperInstance.getItemImportNodeByIdVersion(propertyList.get(2).getId(), "0.1", null,
+                allImportItemNodesList, false);
+        Property imporRecordProperty = theVersionNode.getItemRecord().getProperty();
+        Property property3 = propertyList.get(3);
+        Assert.assertEquals(property3.getId(), imporRecordProperty.getId());
+        Assert.assertEquals("0.1", imporRecordProperty.getVersion());
+        theVersionNode = helperInstance.getItemImportNodeByIdVersion(propertyList.get(2).getId(), "0.7", null,
+                allImportItemNodesList, false);
+        imporRecordProperty = theVersionNode.getItemRecord().getProperty();
+        Assert.assertEquals(property3.getId(), imporRecordProperty.getId());
+        Assert.assertEquals("0.7", imporRecordProperty.getVersion());
+
+        Property routineProperty = propertyList.get(4);
+        ItemImportNode routineImportNode = helperInstance.getItemImportNodeByIdVersion(routineProperty.getLabel(), "0.1", null,
+                allImportItemNodesList, true);
+        Property routineImportProperty = routineImportNode.getItemRecord().getProperty();
+        Assert.assertEquals(routineProperty.getId(), routineImportProperty.getId());
+        Assert.assertEquals(routineProperty.getLabel(), routineImportProperty.getLabel());
+        Assert.assertEquals("0.1", routineImportProperty.getVersion());
+        routineImportNode = helperInstance.getItemImportNodeByIdVersion(routineProperty.getLabel(), "0.7", null,
+                allImportItemNodesList, true);
+        routineImportProperty = routineImportNode.getItemRecord().getProperty();
+        Assert.assertEquals(routineProperty.getId(), routineImportProperty.getId());
+        Assert.assertEquals(routineProperty.getLabel(), routineImportProperty.getLabel());
+        Assert.assertEquals("0.7", routineImportProperty.getVersion());
+    }
+
     @Test
     public void testGetLatestVersionItemImportNode() {
         ItemImportNode latestVersionNode = helperInstance.getLatestVersionItemImportNode(propertyList.get(2).getId(),
-                null, allImportItemNodesList);
+                null, allImportItemNodesList, false);
         Property latestVersionProperty = latestVersionNode.getItemRecord().getProperty();
         Property property3 = propertyList.get(3);
         Assert.assertEquals(latestVersionProperty.getId(), property3.getId());
         Assert.assertEquals(latestVersionProperty.getVersion(), property3.getVersion());
+
+        ItemImportNode latestRoutineNode = helperInstance.getLatestVersionItemImportNode(propertyList.get(4).getLabel(), null,
+                allImportItemNodesList, true);
+        Property latestRoutineProperty = latestRoutineNode.getItemRecord().getProperty();
+        Property property5 = propertyList.get(5);
+        Assert.assertEquals(latestRoutineProperty.getId(), property5.getId());
+        Assert.assertEquals(latestRoutineProperty.getLabel(), property5.getLabel());
+        Assert.assertEquals(latestRoutineProperty.getVersion(), property5.getVersion());
     }
 
     private void createRelations(List<Property> propertyList) {

test/plugins/org.talend.repository.metadata.test/META-INF/MANIFEST.MF

@@ -5,3 +5,4 @@ Bundle-SymbolicName: org.talend.repository.metadata.test
 Bundle-Version: 7.3.1.qualifier
 Fragment-Host: org.talend.repository.metadata
 Require-Bundle: org.talend.testutils
+Import-Package: org.apache.commons.io

test/plugins/org.talend.repository.metadata.test/src/org/talend/repository/ui/wizards/metadata/table/database/ConnectionUUIDHelperTest.java

@@ -21,8 +21,10 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
+import java.nio.charset.StandardCharsets;
 import java.util.Set;
 
+import org.apache.commons.io.FileUtils;
 import org.eclipse.core.runtime.FileLocator;
 import org.eclipse.core.runtime.Path;
 import org.eclipse.emf.common.util.URI;
@@ -42,6 +44,7 @@ import org.talend.cwm.helper.ConnectionHelper;
 import org.talend.cwm.helper.PackageHelper;
 import org.talend.model.emf.CwmResource;
 import org.talend.utils.io.FilesUtils;
+
 import orgomg.cwm.objectmodel.core.Package;
 
 /**
@@ -170,7 +173,7 @@ public class ConnectionUUIDHelperTest {
 
         File testDataFile = getTestDataFile(TEST_DB_MYSQL_ITEM);
         assertNotNull(testDataFile);
-        assertEquals(FilesUtils.getChecksumAlder32(testDataFile), FilesUtils.getChecksumAlder32(copiedFile));
+        FileUtils.contentEqualsIgnoreEOL(testDataFile, copiedFile, StandardCharsets.UTF_8.toString());
     }
 
     @Test
@@ -196,7 +199,7 @@ public class ConnectionUUIDHelperTest {
 
         File testRemovedTableFile = getTestDataFile(TEST_DB_MYSQL_REMOVE_TABLE_ITEM);
         assertNotNull(testRemovedTableFile);
-        assertEquals(FilesUtils.getChecksumAlder32(testRemovedTableFile), FilesUtils.getChecksumAlder32(removedFile));
+        FileUtils.contentEqualsIgnoreEOL(testRemovedTableFile, removedFile, StandardCharsets.UTF_8.toString());
     }
 
     @Test
@@ -219,7 +222,7 @@ public class ConnectionUUIDHelperTest {
         originalItemResourse.save(changedFos, null);
 
         File testDataFile = getTestDataFile(TEST_DB_MYSQL_DIFF_LABEL_ITEM);
-        assertEquals(FilesUtils.getChecksumAlder32(testDataFile), FilesUtils.getChecksumAlder32(changedFile));
+        FileUtils.contentEqualsIgnoreEOL(testDataFile, changedFile, StandardCharsets.UTF_8.toString());
     }
 
 }