fix(TUP-37425): Performance: Not Responding while Retrieving schema and the database contains many tables (#5949)

* fix(TUP-37425): fix retrieve schema

* fix(TUP-37425): check NPE

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/context/ContextUtils.java

@@ -951,14 +951,6 @@ public class ContextUtils {
             params.add(param);
         }
 
-        public boolean remove(Item item, String param) {
-            Set<String> params = map.get(item);
-            if (params != null && params.contains(param)) {
-                return params.remove(param);
-            }
-            return false;
-        }
-
         @SuppressWarnings("unchecked")
         public Set<String> get(Item item) {
             Set<String> params = map.get(item);

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/TalendPropertiesUtil.java

@@ -79,8 +79,4 @@ public final class TalendPropertiesUtil {
     public static String getProductApp() {
         return System.getProperty(PROD_APP);
     }
-
-    public static boolean isEnabledUseShortJobletName() {
-        return isEnabled("talend.job.build.useShortJobletName"); //$NON-NLS-1$
-    }
 }

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/proposal/RoutinesFunctionProposal.java

@@ -78,7 +78,6 @@ public class RoutinesFunctionProposal implements IContentProposal {
         message += Messages.getString("RoutinesFunctionProposal.CreatedBy");
         message += Messages.getString("RoutinesFunctionProposal.ReturnType");
         message += Messages.getString("RoutinesFunctionProposal.VariableName");
-        message = message.replaceAll("\n", System.getProperty("line.separator", "\n")); // for display on Windows platform
 
         MessageFormat format = new MessageFormat(message);
         Object[] args = new Object[] { function.getDescription(),

main/plugins/org.talend.core.ui/src/main/java/org/talend/core/ui/token/TokenInforUtil.java

@@ -134,7 +134,7 @@ public final class TokenInforUtil {
                     targetArray = new JSONArray();
                     Map<String,List<JSONObject>> objectMap = new HashMap<String,List<JSONObject>>();
                     for (Object obj : data) {
-                        if((obj instanceof JSONObject) && ((JSONObject)obj).has("component_name") && ((JSONObject)obj).get("component_name")!=null){//$NON-NLS-1$
+                        if((obj instanceof JSONObject) && ((JSONObject)obj).get("component_name")!=null){//$NON-NLS-1$
                             List<JSONObject> dataList = new ArrayList<JSONObject>();
                             String componentName = (String) ((JSONObject)obj).get("component_name");//$NON-NLS-1$
                             if(objectMap.containsKey(componentName)){

main/plugins/org.talend.designer.maven.tos/resources/components/pom.xml

@@ -42,7 +42,7 @@
 			<dependency>
                 <groupId>org.json</groupId>
                 <artifactId>json</artifactId>
-                <version>20230227</version>
+                <version>20140107</version>
             </dependency>
             <dependency>
                 <groupId>com.fasterxml.jackson.dataformat</groupId>

main/plugins/org.talend.designer.maven.tos/resources/tacokit/pom.xml

@@ -10,7 +10,7 @@
 	<artifactId>studio-tacokit-dependencies</artifactId>
 	<packaging>pom</packaging>
 	<properties>
-		<tacokit.components.version>1.27.22</tacokit.components.version>
+		<tacokit.components.version>1.27.19</tacokit.components.version>
 	</properties>
 	<repositories>
 		<repository>

main/plugins/org.talend.designer.maven.tos/resources/tcompv1/pom.xml

@@ -11,7 +11,7 @@
     <packaging>pom</packaging>
 
     <properties>
-        <tcomp.version>1.38.9</tcomp.version>
+        <tcomp.version>1.38.8</tcomp.version>
         <slf4j.version>1.7.32</slf4j.version>
         <log4j2.version>2.17.1</log4j2.version>
         <reload4j.version>1.2.19</reload4j.version>

main/plugins/org.talend.librariesmanager/resources/java/routines/TalendDate.java

@@ -467,13 +467,13 @@ public class TalendDate {
      *
      * {Category} TalendDate
      *
-     * {param} String("2008/11/24 12:15:25") string : date represent in string
+     * {param} String("") string : date represent in string
      *
-     * {param} String("yyyy/MM/dd HH:mm:ss") pattern : date pattern
+     * {param} String("yyyy-MM-dd") pattern : date pattern
      *
-     * {param} int(5) nb : the added value
+     * {param} int(addValue) nb : the added value
      *
-     * {param} String("dd") dateType : the part to add
+     * {param} date("MM") dateType : the part to add
      *
      * {examples}
      *
@@ -1216,19 +1216,8 @@ public class TalendDate {
     }
 
     /**
-     * Format date to mssql 2008 type datetimeoffset ISO 8601 string with local time zone format string : yyyy-MM-dd
-     * HH:mm:ss.SSSXXX (JDK7 support it)
-     *
-     * @param date the time value to be formatted into a time string.
-     * @return the formatted time string.
-     *
-     * {talendTypes} String
-     *
-     * {Category} TalendDate
-     *
-     * {param} date(new Date()) date : the time value to be formatted into a time string
-     *
-     * {example} formatDatetimeoffset(new Date()) #
+     * format date to mssql 2008 type datetimeoffset ISO 8601 string with local time zone format string : yyyy-MM-dd
+     * HH:mm:ss.SSSXXX(JDK7 support it)
      */
     public static String formatDatetimeoffset(Date date) {
         String dateString = formatDate("yyyy-MM-dd HH:mm:ss.SSSZ", date);// keep the max precision in java
@@ -1357,28 +1346,14 @@ public class TalendDate {
     }
 
     /**
-     * Convert a formatted string to date
-     * 
+     *
      * @param string Must be a string datatype. Passes the values that you want to convert.
      * @param format Enter a valid TO_DATE format string. The format string must match the parts of the string argument
-     * default format is "MM/DD/yyyy HH:mm:ss.sss" if not specified.
-     * 
+     * 		  default formate is "MM/DD/yyyy HH:mm:ss.sss" if not specified.
      * @return Date
      * @throws ParseException
-     * 
-     * {talendTypes} Date
-     *
-     * {Category} TalendDate
-     * 
-     * {param} String("2015-11-21 13:23:45") string : string Must be a string datatype. Passes the values that you want
-     * to convert.
-     *
-     * {param} String("yyyy-MM-dd HH:mm:ss") format : Enter a valid TO_DATE format string. The format string must match
-     * the parts of the string argument default format is "MM/DD/yyyy HH:mm:ss.sss" if not specified.
-     *
-     *
-     * {example} TO_DATE("1464576463231", "J") #Mon May 30 10:47:43 CST 2016 {example} TO_DATE("2015-11-21
-     * 13:23:45","yyyy-MM-dd HH:mm:ss") #Sat Nov 21 13:23:45 CST 2015
+     * {example} TO_DATE("1464576463231", "J") #Mon May 30 10:47:43 CST 2016
+     * {example} TO_DATE("2015-11-21 13:23:45","yyyy-MM-dd HH:mm:ss") #Sat Nov 21 13:23:45 CST 2015
      *
      */
     public static Date TO_DATE(String string, String format) throws ParseException {
@@ -1399,24 +1374,6 @@ public class TalendDate {
 
     }
 
-    /**
-     * Convert a formatted string to date with default format as ""MM/DD/yyyy HH:mm:ss.sss"
-     * 
-     * @param string Must be a string datatype. Passes the values that you want to convert.
-     * @return Date
-     * @throws ParseException
-     * 
-     * {talendTypes} Date
-     *
-     * {Category} TalendDate
-     * 
-     * {param} String("11/21/2015 13:23:45.111") string : string Must be a string datatype. Passes the values that you
-     * want to convert.
-     *
-     * {example} TO_DATE("11/21/2015 13:23:45.111") #Sat Nov 21 13:23:45.111 CST 2015
-     *
-     */
-
     public static Date TO_DATE(String string) throws ParseException {
         return TO_DATE(string, null);
     }
@@ -1453,25 +1410,13 @@ public class TalendDate {
     }
 
     /**
-     * Add values to the specified portion of the date
-     * 
-     * @param date Passes the values you want to change
-     * @param format A format string specifying the portion of the date value you want to change.For example, 'mm'.
-     * @param amount An integer value specifying the amount of years, months, days, hours, and so on by which you want
-     * to change the date value.
-     * @return Date NULL if a null value is passed as an argument to the function.
-     * @throws ParseException
-     * 
-     * {talendTypes} Date
      *
-     * {Category} TalendDate
-     * 
-     * {param} Date(new Date()) date :
-     * 
-     * {param} String("HH") format :
-     * 
-     * {param} int(2) amount :
-     * 
+     * @param date  Passes the values you want to change
+     * @param format A format string specifying the portion of the date value you want to change.For example, 'mm'.
+     * @param amount An integer value specifying the amount of years, months, days, hours,
+     * 				and so on by which you want to change the date value.
+     * @return Date  NULL if a null value is passed as an argument to the function.
+     * @throws ParseException
      * {example} ADD_TO_DATE(new Date(1464576463231l), "HH",2) #Mon May 30 12:47:43 CST 2016
      */
     public static Date ADD_TO_DATE(Date date, String format, int amount) throws ParseException{
@@ -1540,21 +1485,10 @@ public class TalendDate {
     }
 
     /**
-     * Convert a Date to a formatted character string.
      *
-     * @param date the date value you want to convert to character strings.
-     * @param format the format of the return value,
-     * @return String. NULL if a value passed to the function is NULL.
-     *
-     * {talendTypes} String
-     *
-     * {Category} TalendDate
-     *
-     * {param} Date(new Date()) date : the date value you want to convert to character strings.
-     * 
-     * {param} String("MM/DD/YYYY HH24:MI:SS") format : the format of the return value,
-     *
-     * {example} TO_CHAR(new Date(),"MM/DD/YYYY HH24:MI:SS") #
+     * @param date  Date/Time datatype. Passes the date values you want to convert to character strings.
+     * @param format   Enter a valid TO_CHAR format string. The format string defines the format of the return value,
+     * @return  String.   NULL if a value passed to the function is NULL.
      */
 
     public static String TO_CHAR(Date date, String format) {

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/AbstractTableBuilder.java

@@ -19,6 +19,8 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
 
+import metadata.managment.i18n.Messages;
+
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 import org.talend.cwm.helper.ColumnSetHelper;
@@ -27,7 +29,6 @@ import org.talend.metadata.managment.utils.MetadataConnectionUtils;
 import org.talend.utils.sql.metadata.constants.GetTable;
 import org.talend.utils.sql.metadata.constants.TableType;
 
-import metadata.managment.i18n.Messages;
 import orgomg.cwm.resource.relational.NamedColumnSet;
 
 /**
@@ -176,9 +177,9 @@ public abstract class AbstractTableBuilder<T extends NamedColumnSet> extends Cwm
         String tableComment = tablesSet.getString(GetTable.REMARKS.name());
         if (StringUtils.isBlank(tableComment)) {
             String dbProductName = getConnectionMetadata(connection).getDatabaseProductName();
-            String selectRemarkOnTable = MetadataConnectionUtils.getCommonQueryStr(dbProductName);
+            String selectRemarkOnTable = MetadataConnectionUtils.getCommonQueryStr(dbProductName, tableName);
             if (selectRemarkOnTable != null) {
-                tableComment = executeGetCommentStatement(selectRemarkOnTable, tableName);
+                tableComment = executeGetCommentStatement(selectRemarkOnTable);
             }
         }
         return tableComment;

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/core/model/metadata/builder/database/CwmBuilder.java

@@ -14,14 +14,14 @@ package org.talend.core.model.metadata.builder.database;
 
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
-import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-
-import org.apache.log4j.Logger;
+import java.sql.Statement;
 
 import metadata.managment.i18n.Messages;
 
+import org.apache.log4j.Logger;
+
 /**
  * @author scorreia
  *
@@ -73,48 +73,14 @@ abstract class CwmBuilder {
      */
     protected String executeGetCommentStatement(String queryStmt) {
         String comment = null;
-        PreparedStatement statement = null;
+        Statement statement = null;
         ResultSet resultSet = null;
         try {
-            statement = connection.prepareStatement(queryStmt);
-            resultSet = statement.executeQuery();
-
-            if (resultSet != null) {
-                while (resultSet.next()) {
-                    comment = (String) resultSet.getObject(1);
-                }
-            }
-        } catch (SQLException e) {
-            // do nothing here
-        } finally {
-            // -- release resources
-            if (resultSet != null) {
-                try {
-                    resultSet.close();
-                } catch (SQLException e) {
-                    log.error(e, e);
-                }
-            }
-            if (statement != null) {
-                try {
-                    statement.close();
-                } catch (SQLException e) {
-                    log.error(e, e);
-                }
-            }
-        }
-        return comment;
-    }
-
-    protected String executeGetCommentStatement(String queryStmt, String tableName) {
-        String comment = null;
-        PreparedStatement statement = null;
-        ResultSet resultSet = null;
-        try {
-            statement = connection.prepareStatement(queryStmt);
-            statement.setString(1, tableName);
-            resultSet = statement.executeQuery();
+            statement = connection.createStatement();
+            statement.execute(queryStmt);
 
+            // get the results
+            resultSet = statement.getResultSet();
             if (resultSet != null) {
                 while (resultSet.next()) {
                     comment = (String) resultSet.getObject(1);

main/plugins/org.talend.metadata.managment/src/main/java/org/talend/metadata/managment/utils/MetadataConnectionUtils.java

@@ -805,33 +805,6 @@ public class MetadataConnectionUtils {
 
     }
 
-    public static String getCommonQueryStr(String productName) {
-        if (productName == null) {
-            return null;
-        }
-        productName = productName.replaceAll(" ", "_"); //$NON-NLS-1$ //$NON-NLS-2$
-        EDataBaseType eDataBaseType = null;
-        try {
-            eDataBaseType = EDataBaseType.valueOf(productName);
-        } catch (Exception e) {
-            eDataBaseType = EDataBaseType.Microsoft_SQL_Server;
-        }
-        String sqlStr = ""; //$NON-NLS-1$
-        switch (eDataBaseType) {
-        case Oracle:
-            sqlStr = "SELECT COMMENTS FROM USER_TAB_COMMENTS WHERE TABLE_NAME= ? "; //$NON-NLS-1$
-            break;
-        case MySQL:
-            sqlStr = "SELECT TABLE_COMMENT FROM information_schema.TABLES WHERE TABLE_NAME= ? "; //$NON-NLS-1$
-            break;
-        default:
-            sqlStr = null;
-
-        }
-        return sqlStr;
-
-    }
-
     /**
      * get Comment Query Str.
      *

main/plugins/org.talend.migrationTool/src/org/talend/migrationtool/MigrationToolService.java

@@ -23,7 +23,6 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Optional;
-import java.util.stream.Stream;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
@@ -241,14 +240,20 @@ public class MigrationToolService implements IMigrationToolService {
         sortMigrationTasks(toExecute);
 
         final List<MigrationTask> done = new ArrayList<MigrationTask>(storedMigrations);
-        boolean hasTaskToExecute = toExecute.stream()
-                .anyMatch(task -> !task.isDeprecated() && MigrationUtil.findMigrationTask(done, task) == null);
+        int nbMigrationsToDo = 0;
+        for (IProjectMigrationTask task : toExecute) {
+            MigrationTask mgTask = MigrationUtil.findMigrationTask(done, task);
+            if (mgTask == null && !task.isDeprecated()) {
+                nbMigrationsToDo++;
+            }
+        }
 
         // force to redo the migration task for the relations only if user ask for "clean" or if relations is empty
         // or if there is at least another migration to do.
-        if (!beforeLogon && (!RelationshipItemBuilder.INDEX_VERSION.equals(project.getEmfProject().getItemsRelationVersion())
-                || hasTaskToExecute)) {
+        if (!beforeLogon
+                && (!RelationshipItemBuilder.INDEX_VERSION.equals(project.getEmfProject().getItemsRelationVersion()) || nbMigrationsToDo > 0)) {
             // force to redo this migration task, to make sure the relationship is done correctly
+            // done.remove(RELATION_TASK);
             MigrationUtil.removeMigrationTaskById(done, RELATION_TASK);
             RelationshipItemBuilder.getInstance().unloadRelations();
 
@@ -259,37 +264,41 @@ public class MigrationToolService implements IMigrationToolService {
                     RelationshipItemBuilder.JOBLET_RELATION, true);
             // reset
             RelationshipItemBuilder.getInstance().unloadRelations();
-            hasTaskToExecute = true;
-        }
 
-        boolean checkDupContext = !beforeLogon && Boolean.getBoolean("duplicate.context.reference.check");
-        if (!hasTaskToExecute && !checkDupContext) {
+            nbMigrationsToDo++;
+        }
+        if (nbMigrationsToDo == 0) {
             return;
         }
 
-        if (checkDupContext) {
-            MigrationUtil.removeMigrationTaskById(done,
-                    "org.talend.repository.model.migration.RemoveDuplicateContextReferencesMigrationTask");
-        }
-        if (hasTaskToExecute) {
-            // force execute migration in case user copy-past items with diffrent path on the file system and refresh
-            // the studio,it may cause bug TDI-19229
-            MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.FixProjectResourceLink");
-            // force to re-generate all job poms
-            MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.GenerateJobPomMigrationTask");
+        // force execute migration in case user copy-past items with diffrent path on the file system and refresh
+        // the studio,it may cause bug TDI-19229
+        MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.FixProjectResourceLink");
+        // force to re-generate all job poms
+        MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.GenerateJobPomMigrationTask");
 
-            if (beforeLogon) {
-                // for every migration, force reset to default maven template
-                MigrationUtil.removeMigrationTaskById(done,
-                        "org.talend.repository.model.migration.ResetMavenTemplateMigrationTask");
+        if (beforeLogon) {
+            // for every migration, force reset to default maven template
+            MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.ResetMavenTemplateMigrationTask");
+        }
+
+        boolean haveAnyBinFolder = false; // to avoid some problems of migration, sometimes
+        for (ERepositoryObjectType type : (ERepositoryObjectType[]) ERepositoryObjectType.values()) {
+            if (!type.hasFolder()) {
+                continue;
             }
-            boolean hasBinFolder = Stream.of((ERepositoryObjectType[]) ERepositoryObjectType.values())
-                    .filter(type -> type.hasFolder()).map(ERepositoryObjectType::getFolderName).filter(StringUtils::isNotBlank)
-                    .map(folderName -> fsProject.getFolder(folderName))
-                    .anyMatch(folder -> folder.exists() && folder.getFolder("bin").exists());
-            if (hasBinFolder) {
-                MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.RemoveBinFolderMigrationTask");
+            String folderName = ERepositoryObjectType.getFolderName(type);
+            if (folderName == null || "".equals(folderName)) {
+                continue;
             }
+            IFolder folder = fsProject.getFolder(folderName);
+            if (folder.exists() && folder.getFolder("bin").exists()) { //$NON-NLS-1$
+                haveAnyBinFolder = true;
+                break;
+            }
+        }
+        if (haveAnyBinFolder) {
+            MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.RemoveBinFolderMigrationTask");
         }
 
         final SubProgressMonitor subProgressMonitor = new SubProgressMonitor(monitorWrap, toExecute.size());

main/plugins/org.talend.repository.metadata/src/main/java/org/talend/repository/ui/wizards/metadata/table/database/SelectorTableForm.java

@@ -1181,10 +1181,6 @@ public class SelectorTableForm extends AbstractForm {
                         }
                         if (canAdd) {
                             tableNodes.add(schemaNode);
-                            if (schemaNode.getValue() != null && StringUtils.isEmpty(schemaNode.getValue().trim())) {
-                                List<MetadataTable> tableList = new ArrayList<MetadataTable>();
-                                retrieveAllSubNodes(schemaNode, tableList);
-                            }
                         }
                     }
                 }

main/plugins/org.talend.repository.view.example/components/tExampleComponent/tExampleComponent_java.xml

@@ -201,8 +201,8 @@ SHOW_IF="USE_FILE_AMBIGUOUS=='true'">
 	<CODEGENERATION>
 		<IMPORTS>
 			<IMPORT NAME="Talen File Enhanced"
-				MODULE="talend_file_enhanced-1.3.jar"
-				MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.3" REQUIRED="true" />
+				MODULE="talend_file_enhanced-1.1.jar"
+				MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" REQUIRED="true" />
 			<IMPORT NAME="Talend_CSV" MODULE="talendcsv.jar"
 				MVN="mvn:org.talend.libraries/talendcsv/6.0.0"
 				REQUIRED="true" />