Compare commits

...

23 Commits

Author SHA1 Message Date
zyuan-talend
7c8614b4ac fix(TUP-39467): CVE-2022-29599,maven-shared-utils:0.9.0. (#6272) 2023-06-29 14:38:41 +08:00
jzhao
baacc284fd chore(TDI-49988):bump TCK connector 1.27.24 for R2023-07 (#6263) 2023-06-25 19:22:28 +08:00
sbliu
9d0fdc4532 chore(TUP-39233) upgrade json-smart, accessors-smart to 2.4.11 (#6217) 2023-06-20 16:19:35 +08:00
Zhiwei Xue
f3ad09604f fix(TUP-38235):Job in a project that references other projects load very (#6240)
slowly in Studio
2023-06-20 09:47:52 +08:00
jzhao
4615ebdb75 chore(TDI-49841):bump TCK connector 1.27.23 for R2023-06 (#6198) 2023-05-22 11:37:34 +08:00
msjian
5616d65272 fix(TDQ-21121): upgrade snowflake connection driver (#6141) 2023-05-11 15:09:00 +08:00
Abdulhamid Rashankar
ca85dd4b73 APPINT-35490:App & API Integration - Studio Metrics gathering: build types & APIs usage (#6159) 2023-05-02 14:07:02 +02:00
sbliu
11d3c32066 chore(TUP-38551) json-smart:2.4.7 ( CVE-2023-1370) (#6097)
* chore(TUP-38551) json-smart:2.4.7 ( CVE-2023-1370)

* update json-smart version 2.4.10 to 2.4.9

* update maven url index for upgrade json-smart
2023-04-26 10:28:20 +08:00
wang wei
a866707aae fix(TDI-49452): CVE-2022-45143,org.apache.tomcat:tomcat-catalina:9.0.68 (#6122) 2023-04-24 15:35:21 +08:00
wang wei
53e052989f fix(TDI-49546): CVE-2022-45688, org.json:json:(<=20220924) (#6065) 2023-04-24 14:08:06 +08:00
jzhao
f52c747409 chore(TDI-49727):Bump TCK connector 1.27.22 for R2023-05 (#6139) 2023-04-23 12:04:10 +08:00
Zhiwei Xue
6d36ff79bb fix(TUP-38316):[801 CI/CD] Duplicate Parameter error in the artifact and (#6081)
creating the task
2023-04-20 18:33:36 +08:00
Jane Ding
c362aa776c fix(TUP-37227):Issue building when items names too long (#6067)
https://jira.talendforge.org/browse/TUP-37227
2023-03-30 18:15:46 +08:00
vyu-talend
318724d32c chore(TDI-49458):bump file enhanced. (#6077) 2023-03-30 16:21:46 +08:00
Oleksandr Zhelezniak
4b377af1aa chore(TDI-49600): bump connectors 1.27.21 2023-03-24 15:09:35 +02:00
Jane Ding
695dcba405 Revert "fix(TUP-37227):Issue building when items names too long (#5941)" (#6019)
This reverts commit 8d3832fdfb.
2023-03-02 09:44:55 +08:00
jiezhang-tlnd
4dc7e4e82f fix(TUP-37874)SQL inject risk (#6009) 2023-02-23 15:13:01 +08:00
jiezhang-tlnd
4c819add92 fix(TUP-37874)SQL inject risk (#5970)
* fix(TUP-37874)SQL inject risk

* SQL inject risk
2023-02-22 18:18:23 +08:00
zyuan-talend
9c0f5e966d fix(TUP-35060): tWarn does not show the TalendDate.TO_CHAR method. (#5969) 2023-02-21 17:45:05 +08:00
Oleksandr Zhelezniak
da5b7979fd chore(TDI-49419): bump connectors 1.27.20 2023-02-16 17:14:14 +02:00
Jane Ding
8d3832fdfb fix(TUP-37227):Issue building when items names too long (#5941)
* fix(TUP-37227):Issue building when items names too long
https://jira.talendforge.org/browse/TUP-37227

* fix(TUP-37227):Issue building when items names too long
https://jira.talendforge.org/browse/TUP-37227

* fix(TUP-37227):Issue building when items names too long
https://jira.talendforge.org/browse/TUP-37227

* fix(TUP-37227):Issue building when items names too long
https://jira.talendforge.org/browse/TUP-37227
2023-02-15 18:01:04 +08:00
bhe-talendbj
cd8ff90b5b fix(TUP-37920): fix sqllite retrieve tables (#5960) 2023-02-06 17:23:42 +08:00
bhe-talendbj
d98e5ea43c fix(TUP-37425): Performance: Not Responding while Retrieving schema and the database contains many tables (#5948)
* fix(TUP-37425): fix retrieve schema

* fix(TUP-37425): check NPE
2023-01-30 19:38:12 +08:00
28 changed files with 460 additions and 229 deletions

View File

@@ -872,25 +872,38 @@ public abstract class AbstractEMFRepositoryFactory extends AbstractRepositoryFac
@Override
public IRepositoryViewObject getLastVersion(Project project, String id, String relativeFolder, ERepositoryObjectType type)
throws PersistenceException {
List<IRepositoryViewObject> serializableAllVersion = null;
Object fullFolder = getFullFolder(project, type, relativeFolder);
serializableAllVersion = getSerializableFromFolder(project, fullFolder, id, type, false, false, true, true);
List<IRepositoryViewObject> serializableAllVersion = new ArrayList<>();
if (lastFolderForItemMap.containsKey(id)) {
ERepositoryObjectType itemType = lastRepositoryTypeForItemMap.get(id);
String currentPath = lastFolderForItemMap.get(id);
Object fullFolder = getFullFolder(project, itemType, currentPath);
try {
if (fullFolder != null && (fullFolder instanceof FolderItem || ((IFolder) fullFolder).exists())) {
serializableAllVersion.addAll(getSerializableFromFolder(project, fullFolder, id, itemType, false, false, true, true));
}
} catch (PersistenceException e) {
// do nothing.
// if any exception happen or can't find the item, just try to look for it everywhere.
}
}
if (serializableAllVersion.isEmpty()) {
// look in all folders for this item type
serializableAllVersion = getSerializableFromFolder(project, fullFolder, id, type, false, true, true, true, true);
Object fullFolder = getFullFolder(project, type, relativeFolder);
serializableAllVersion = getSerializableFromFolder(project, fullFolder, id, type, false, false, true, true);
if (serializableAllVersion.isEmpty()) {
// look in all folders for this item type
serializableAllVersion = getSerializableFromFolder(project, fullFolder, id, type, false, true, true, true, true);
}
}
int size = serializableAllVersion.size();
if (size > 1) {
String message = getItemsMessages(serializableAllVersion, size);
throw new PersistenceException(Messages.getString(
"AbstractEMFRepositoryFactory.presistenceException.OnlyOneOccurenceMustbeFound", message)); //$NON-NLS-1$
} else if (size == 1) {
return serializableAllVersion.get(0);
} else {
return null;
}
if (size == 1) {
return serializableAllVersion.get(0);
}
return null;
}
protected void computePropertyMaxInformationLevel(Property property) {

View File

@@ -88,8 +88,8 @@ public enum EDatabaseVersion4Drivers {
"Microsoft", "MSSQL_PROP", //$NON-NLS-1$ //$NON-NLS-2$
new String[] { "mssql-jdbc.jar", "slf4j-api-1.7.25.jar", "slf4j-log4j12-1.7.25.jar", "adal4j-1.6.7.jar", //$NON-NLS-1$
"commons-lang3-3.10.jar", "commons-codec-1.14.jar", "gson-2.8.9.jar", "oauth2-oidc-sdk-9.7.jar",
"json-smart-2.4.7.jar", "nimbus-jose-jwt-9.22.jar", "javax.mail-1.6.2.jar", "reload4j-1.2.19.jar",
"accessors-smart-2.4.7.jar", "asm-9.1.jar", "content-type-2.1.jar" })),
"json-smart-2.4.11.jar", "nimbus-jose-jwt-9.22.jar", "javax.mail-1.6.2.jar", "reload4j-1.2.19.jar",
"accessors-smart-2.4.11.jar", "asm-9.5.jar", "content-type-2.1.jar" })),
VERTICA_9(new DbVersion4Drivers(EDatabaseTypeName.VERTICA, "VERTICA 9.X", "VERTICA_9_0", "vertica-jdbc-9.3.1-0.jar")), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
VERTICA_7_1_X(new DbVersion4Drivers(EDatabaseTypeName.VERTICA, "VERTICA 7.1.X (Deprecated)", "VERTICA_7_1_X", "vertica-jdbc-7.1.2-0.jar")), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

View File

@@ -495,11 +495,16 @@ public class ContextUtils {
}
return itemMap;
}
private static Set<String> missingContexts = new HashSet<>();
public static void clearMissingContextCache() {
missingContexts.clear();
}
/**
*
* get the repository context item,now contextId can be either joblet node or context node.
*/
* get the repository context item, now contextId can be either joblet node or context node.
*/
public static Item getRepositoryContextItemById(String contextId) {
if (IContextParameter.BUILT_IN.equals(contextId)) {
return null;
@@ -507,6 +512,9 @@ public class ContextUtils {
if (checkObject(contextId)) {
return null;
}
if (missingContexts.contains(contextId)) {
return null;
}
List<ERepositoryObjectType> possibleTypes = new ArrayList<ERepositoryObjectType>();
possibleTypes.add(ERepositoryObjectType.CONTEXT);
@@ -521,6 +529,8 @@ public class ContextUtils {
return item;
}
}
missingContexts.add(contextId);
ExceptionHandler.log("Can't find Context item[id=" + contextId + "].");
} catch (PersistenceException e) {
ExceptionHandler.process(e);
}
@@ -831,6 +841,7 @@ public class ContextUtils {
ItemContextLink itemContextLink) {
Map<String, String> renamedMap = new HashMap<String, String>();
Map<String, Item> tempItemMap = new HashMap<String, Item>();
clearMissingContextCache();
for (ContextType contextType : contextTypeList) {
for (Object obj : contextType.getContextParameter()) {
if (obj instanceof ContextParameterType) {
@@ -893,6 +904,7 @@ public class ContextUtils {
*/
public static Map<String, String> calculateRenamedMapFromLinkFile(String projectLabel, String itemId,
List<IContext> contextList) {
clearMissingContextCache();
Map<String, String> renamedMap = new HashMap<String, String>();
Map<String, Item> idToItemMap = new HashMap<String, Item>();
try {
@@ -951,6 +963,14 @@ public class ContextUtils {
params.add(param);
}
public boolean remove(Item item, String param) {
Set<String> params = map.get(item);
if (params != null && params.contains(param)) {
return params.remove(param);
}
return false;
}
@SuppressWarnings("unchecked")
public Set<String> get(Item item) {
Set<String> params = map.get(item);

View File

@@ -299,6 +299,7 @@ public class JobContextManager implements IContextManager {
}
List<ContextItem> contextItemList = ContextUtils.getAllContextItem();
boolean setDefault = false;
ContextUtils.clearMissingContextCache();
for (int i = 0; i < contextTypeList.size(); i++) {
contextType = (ContextType) contextTypeList.get(i);
String name = contextType.getName();
@@ -526,6 +527,7 @@ public class JobContextManager implements IContextManager {
EList newcontextTypeList = new BasicEList();
Map<String, Item> idToItemMap = new HashMap<String, Item>();
ContextUtils.clearMissingContextCache();
for (int i = 0; i < listContext.size(); i++) {
IContext context = listContext.get(i);
String contextGroupName = renameGroupContext.get(context);

View File

@@ -51,6 +51,7 @@ public abstract class AbstractItemContextLinkService implements IItemContextLink
itemContextLink.setItemId(itemId);
Map<String, Item> tempCache = new HashMap<String, Item>();
if (contextTypeList != null && contextTypeList.size() > 0) {
ContextUtils.clearMissingContextCache();
for (Object object : contextTypeList) {
if (object instanceof ContextType) {
ContextType jobContextType = (ContextType) object;

View File

@@ -206,6 +206,7 @@ public class ContextLinkService {
Map<String, Map<String, String>> changedContextParameterId) throws PersistenceException {
List<Relation> relationList = RelationshipItemBuilder.getInstance()
.getItemsHaveRelationWith(sourceId, RelationshipItemBuilder.LATEST_VERSION, false);
ContextUtils.clearMissingContextCache();
for (Relation relation : relationList) {
String id = relation.getId();
IFile linkFile = calContextLinkFile(ProjectManager.getInstance().getCurrentProject().getTechnicalLabel(), id);

View File

@@ -79,4 +79,8 @@ public final class TalendPropertiesUtil {
public static String getProductApp() {
return System.getProperty(PROD_APP);
}
public static boolean isEnabledUseShortJobletName() {
return isEnabled("talend.job.build.useShortJobletName"); //$NON-NLS-1$
}
}

View File

@@ -648,6 +648,7 @@ public class ContextNebulaGridComposite extends AbstractContextTabEditComposite
helper.initHelper(contextManager);
Map<String, Item> items = new HashMap<String, Item>();
boolean needRefresh = false;
ContextUtils.clearMissingContextCache();
for (IContextParameter param : contextManager.getDefaultContext().getContextParameterList()) {
if (!param.isBuiltIn()) {
String source = param.getSource();

View File

@@ -522,6 +522,7 @@ public class SelectRepositoryContextDialog extends SelectionDialog {
// remove the params which is unchecked
Set<String> jobletIds = new HashSet<String>();
Set<String> chekedIds = new HashSet<String>();
ContextUtils.clearMissingContextCache();
for (IContextParameter param : existParas) {
if (param.isBuiltIn()) {
continue;

View File

@@ -185,6 +185,7 @@ public class AddRepositoryContextGroupCommand extends Command {
// remove the params which is unchecked
Set<String> jobletIds = new HashSet<String>();
Set<String> chekedIds = new HashSet<String>();
ContextUtils.clearMissingContextCache();
for (IContextParameter param : existParas) {
if (param.isBuiltIn()) {
continue;

View File

@@ -92,6 +92,7 @@ public class ContextNatTableUtils {
List<ContextTableTabParentModel> output = new ArrayList<ContextTableTabParentModel>();
if (!contextDatas.isEmpty()) {
int i = 0;
ContextUtils.clearMissingContextCache();
for (IContextParameter para : contextDatas) {
String sourceId = para.getSource();
if (IContextParameter.BUILT_IN.equals(sourceId)) {

View File

@@ -21,6 +21,7 @@ import org.eclipse.ui.IViewPart;
import org.eclipse.ui.IWorkbenchPage;
import org.eclipse.ui.IWorkbenchPart;
import org.eclipse.ui.PlatformUI;
import org.talend.core.model.context.ContextUtils;
import org.talend.core.ui.branding.IBrandingConfiguration;
/**
@@ -51,6 +52,7 @@ public class Contexts {
public void run() {
if (cxtView != null) {
updateTitle(cxtView);
ContextUtils.clearMissingContextCache();
cxtView.refresh(part);
}
}

View File

@@ -78,6 +78,7 @@ public class RoutinesFunctionProposal implements IContentProposal {
message += Messages.getString("RoutinesFunctionProposal.CreatedBy");
message += Messages.getString("RoutinesFunctionProposal.ReturnType");
message += Messages.getString("RoutinesFunctionProposal.VariableName");
message = message.replaceAll("\n", System.getProperty("line.separator", "\n")); // for display on Windows platform
MessageFormat format = new MessageFormat(message);
Object[] args = new Object[] { function.getDescription(),

View File

@@ -134,7 +134,7 @@ public final class TokenInforUtil {
targetArray = new JSONArray();
Map<String,List<JSONObject>> objectMap = new HashMap<String,List<JSONObject>>();
for (Object obj : data) {
if((obj instanceof JSONObject) && ((JSONObject)obj).get("component_name")!=null){//$NON-NLS-1$
if((obj instanceof JSONObject) && ((JSONObject)obj).has("component_name") && ((JSONObject)obj).get("component_name")!=null){//$NON-NLS-1$
List<JSONObject> dataList = new ArrayList<JSONObject>();
String componentName = (String) ((JSONObject)obj).get("component_name");//$NON-NLS-1$
if(objectMap.containsKey(componentName)){

View File

@@ -126,7 +126,7 @@
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<version>2.22.1</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>

View File

@@ -42,7 +42,7 @@
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20140107</version>
<version>20230227</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>

View File

@@ -10,7 +10,7 @@
<artifactId>studio-tacokit-dependencies</artifactId>
<packaging>pom</packaging>
<properties>
<tacokit.components.version>1.27.19</tacokit.components.version>
<tacokit.components.version>1.27.24</tacokit.components.version>
</properties>
<repositories>
<repository>

View File

@@ -11,7 +11,7 @@
<packaging>pom</packaging>
<properties>
<tcomp.version>1.38.8</tcomp.version>
<tcomp.version>1.38.9</tcomp.version>
<slf4j.version>1.7.32</slf4j.version>
<log4j2.version>2.17.1</log4j2.version>
<reload4j.version>1.2.19</reload4j.version>

View File

@@ -467,13 +467,13 @@ public class TalendDate {
*
* {Category} TalendDate
*
* {param} String("") string : date represent in string
* {param} String("2008/11/24 12:15:25") string : date represent in string
*
* {param} String("yyyy-MM-dd") pattern : date pattern
* {param} String("yyyy/MM/dd HH:mm:ss") pattern : date pattern
*
* {param} int(addValue) nb : the added value
* {param} int(5) nb : the added value
*
* {param} date("MM") dateType : the part to add
* {param} String("dd") dateType : the part to add
*
* {examples}
*
@@ -1216,8 +1216,19 @@ public class TalendDate {
}
/**
* format date to mssql 2008 type datetimeoffset ISO 8601 string with local time zone format string : yyyy-MM-dd
* HH:mm:ss.SSSXXX(JDK7 support it)
* Format date to mssql 2008 type datetimeoffset ISO 8601 string with local time zone format string : yyyy-MM-dd
* HH:mm:ss.SSSXXX (JDK7 support it)
*
* @param date the time value to be formatted into a time string.
* @return the formatted time string.
*
* {talendTypes} String
*
* {Category} TalendDate
*
* {param} date(new Date()) date : the time value to be formatted into a time string
*
* {example} formatDatetimeoffset(new Date()) #
*/
public static String formatDatetimeoffset(Date date) {
String dateString = formatDate("yyyy-MM-dd HH:mm:ss.SSSZ", date);// keep the max precision in java
@@ -1346,14 +1357,28 @@ public class TalendDate {
}
/**
*
* Convert a formatted string to date
*
* @param string Must be a string datatype. Passes the values that you want to convert.
* @param format Enter a valid TO_DATE format string. The format string must match the parts of the string argument
* default formate is "MM/DD/yyyy HH:mm:ss.sss" if not specified.
* default format is "MM/DD/yyyy HH:mm:ss.sss" if not specified.
*
* @return Date
* @throws ParseException
* {example} TO_DATE("1464576463231", "J") #Mon May 30 10:47:43 CST 2016
* {example} TO_DATE("2015-11-21 13:23:45","yyyy-MM-dd HH:mm:ss") #Sat Nov 21 13:23:45 CST 2015
*
* {talendTypes} Date
*
* {Category} TalendDate
*
* {param} String("2015-11-21 13:23:45") string : string Must be a string datatype. Passes the values that you want
* to convert.
*
* {param} String("yyyy-MM-dd HH:mm:ss") format : Enter a valid TO_DATE format string. The format string must match
* the parts of the string argument default format is "MM/DD/yyyy HH:mm:ss.sss" if not specified.
*
*
* {example} TO_DATE("1464576463231", "J") #Mon May 30 10:47:43 CST 2016 {example} TO_DATE("2015-11-21
* 13:23:45","yyyy-MM-dd HH:mm:ss") #Sat Nov 21 13:23:45 CST 2015
*
*/
public static Date TO_DATE(String string, String format) throws ParseException {
@@ -1374,6 +1399,24 @@ public class TalendDate {
}
/**
* Convert a formatted string to date with default format as ""MM/DD/yyyy HH:mm:ss.sss"
*
* @param string Must be a string datatype. Passes the values that you want to convert.
* @return Date
* @throws ParseException
*
* {talendTypes} Date
*
* {Category} TalendDate
*
* {param} String("11/21/2015 13:23:45.111") string : string Must be a string datatype. Passes the values that you
* want to convert.
*
* {example} TO_DATE("11/21/2015 13:23:45.111") #Sat Nov 21 13:23:45.111 CST 2015
*
*/
public static Date TO_DATE(String string) throws ParseException {
return TO_DATE(string, null);
}
@@ -1410,13 +1453,25 @@ public class TalendDate {
}
/**
*
* @param date Passes the values you want to change
* Add values to the specified portion of the date
*
* @param date Passes the values you want to change
* @param format A format string specifying the portion of the date value you want to change.For example, 'mm'.
* @param amount An integer value specifying the amount of years, months, days, hours,
* and so on by which you want to change the date value.
* @return Date NULL if a null value is passed as an argument to the function.
* @param amount An integer value specifying the amount of years, months, days, hours, and so on by which you want
* to change the date value.
* @return Date NULL if a null value is passed as an argument to the function.
* @throws ParseException
*
* {talendTypes} Date
*
* {Category} TalendDate
*
* {param} Date(new Date()) date :
*
* {param} String("HH") format :
*
* {param} int(2) amount :
*
* {example} ADD_TO_DATE(new Date(1464576463231l), "HH",2) #Mon May 30 12:47:43 CST 2016
*/
public static Date ADD_TO_DATE(Date date, String format, int amount) throws ParseException{
@@ -1485,10 +1540,21 @@ public class TalendDate {
}
/**
* Convert a Date to a formatted character string.
*
* @param date Date/Time datatype. Passes the date values you want to convert to character strings.
* @param format Enter a valid TO_CHAR format string. The format string defines the format of the return value,
* @return String. NULL if a value passed to the function is NULL.
* @param date the date value you want to convert to character strings.
* @param format the format of the return value,
* @return String. NULL if a value passed to the function is NULL.
*
* {talendTypes} String
*
* {Category} TalendDate
*
* {param} Date(new Date()) date : the date value you want to convert to character strings.
*
* {param} String("MM/DD/YYYY HH24:MI:SS") format : the format of the return value,
*
* {example} TO_CHAR(new Date(),"MM/DD/YYYY HH24:MI:SS") #
*/
public static String TO_CHAR(Date date, String format) {

View File

@@ -48,9 +48,9 @@
context="plugin:org.talend.metadata.managment"
language="java"
message="Needed for create snowflake connection"
mvn_uri="mvn:net.snowflake/snowflake-jdbc/3.11.0"
name="snowflake-jdbc-3.11.0.jar"
required="false">
mvn_uri="mvn:net.snowflake/snowflake-jdbc/3.13.29"
name="snowflake-jdbc-3.13.29.jar"
required="true">
</libraryNeeded>
<libraryNeeded
context="plugin:org.talend.metadata.managment"
@@ -148,6 +148,33 @@
required="true"
uripath="platform:/plugin/org.talend.libraries.jdbc.hsql/lib/hsqldb.jar">
</libraryNeeded>
<libraryNeeded
context="plugin:org.talend.metadata.managment"
language="java"
message="Needed for create Microsoft SQL Server db connection"
mvn_uri="mvn:net.minidev/json-smart/2.4.11"
name="json-smart-2.4.11.jar"
required="true"
uripath="platform:/plugin/org.talend.libraries.tis.custom/lib/json-smart-2.4.11.jar">
</libraryNeeded>
<libraryNeeded
context="plugin:org.talend.metadata.managment"
language="java"
message="Needed for create Microsoft SQL Server db connection"
mvn_uri="mvn:net.minidev/accessors-smart/2.4.11"
name="accessors-smart-2.4.11.jar"
required="true"
uripath="platform:/plugin/org.talend.libraries.tis.custom/lib/accessors-smart-2.4.11.jar">
</libraryNeeded>
<libraryNeeded
context="plugin:org.talend.metadata.managment"
language="java"
message="Needed for create Microsoft SQL Server db connection"
mvn_uri="mvn:org.ow2.asm/asm/9.5"
name="asm-9.5.jar"
required="true"
uripath="platform:/plugin/org.talend.libraries.tis.custom/lib/asm-9.5.jar">
</libraryNeeded>
</extension>
<extension
point="org.talend.core.migrationTask">

View File

@@ -19,8 +19,6 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import metadata.managment.i18n.Messages;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.talend.cwm.helper.ColumnSetHelper;
@@ -29,6 +27,7 @@ import org.talend.metadata.managment.utils.MetadataConnectionUtils;
import org.talend.utils.sql.metadata.constants.GetTable;
import org.talend.utils.sql.metadata.constants.TableType;
import metadata.managment.i18n.Messages;
import orgomg.cwm.resource.relational.NamedColumnSet;
/**
@@ -177,9 +176,9 @@ public abstract class AbstractTableBuilder<T extends NamedColumnSet> extends Cwm
String tableComment = tablesSet.getString(GetTable.REMARKS.name());
if (StringUtils.isBlank(tableComment)) {
String dbProductName = getConnectionMetadata(connection).getDatabaseProductName();
String selectRemarkOnTable = MetadataConnectionUtils.getCommonQueryStr(dbProductName, tableName);
String selectRemarkOnTable = MetadataConnectionUtils.getCommonQueryStr(dbProductName);
if (selectRemarkOnTable != null) {
tableComment = executeGetCommentStatement(selectRemarkOnTable);
tableComment = executeGetCommentStatement(selectRemarkOnTable, tableName);
}
}
return tableComment;

View File

@@ -14,14 +14,14 @@ package org.talend.core.model.metadata.builder.database;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import metadata.managment.i18n.Messages;
import org.apache.log4j.Logger;
import metadata.managment.i18n.Messages;
/**
* @author scorreia
*
@@ -73,14 +73,48 @@ abstract class CwmBuilder {
*/
protected String executeGetCommentStatement(String queryStmt) {
String comment = null;
Statement statement = null;
PreparedStatement statement = null;
ResultSet resultSet = null;
try {
statement = connection.createStatement();
statement.execute(queryStmt);
statement = connection.prepareStatement(queryStmt);
resultSet = statement.executeQuery();
if (resultSet != null) {
while (resultSet.next()) {
comment = (String) resultSet.getObject(1);
}
}
} catch (SQLException e) {
// do nothing here
} finally {
// -- release resources
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
log.error(e, e);
}
}
if (statement != null) {
try {
statement.close();
} catch (SQLException e) {
log.error(e, e);
}
}
}
return comment;
}
protected String executeGetCommentStatement(String queryStmt, String tableName) {
String comment = null;
PreparedStatement statement = null;
ResultSet resultSet = null;
try {
statement = connection.prepareStatement(queryStmt);
statement.setString(1, tableName);
resultSet = statement.executeQuery();
// get the results
resultSet = statement.getResultSet();
if (resultSet != null) {
while (resultSet.next()) {
comment = (String) resultSet.getObject(1);

View File

@@ -126,7 +126,7 @@ public class ExtractMetaDataUtils {
public static final String SNOWFLAKE = "Snowflake"; //$NON-NLS-1$
public static final String SNOWFLAKE_DRIVER_JAR = "snowflake-jdbc-3.11.0.jar"; //$NON-NLS-1$
public static final String SNOWFLAKE_DRIVER_JAR = "snowflake-jdbc-3.13.29.jar"; //$NON-NLS-1$
private ExtractMetaDataUtils() {
}

View File

@@ -1517,6 +1517,9 @@ public class DBConnectionFillerImpl extends MetadataFillerImpl<DatabaseConnectio
private boolean hasRemarksColumn(ResultSet resultSet) {
try {
if (resultSet == null || resultSet.getMetaData() == null) {
return false;
}
int numOfCols = resultSet.getMetaData().getColumnCount();
for (int i = 1; i < numOfCols + 1; i++) {
String colName = resultSet.getMetaData().getColumnLabel(i);

View File

@@ -805,6 +805,33 @@ public class MetadataConnectionUtils {
}
public static String getCommonQueryStr(String productName) {
if (productName == null) {
return null;
}
productName = productName.replaceAll(" ", "_"); //$NON-NLS-1$ //$NON-NLS-2$
EDataBaseType eDataBaseType = null;
try {
eDataBaseType = EDataBaseType.valueOf(productName);
} catch (Exception e) {
eDataBaseType = EDataBaseType.Microsoft_SQL_Server;
}
String sqlStr = ""; //$NON-NLS-1$
switch (eDataBaseType) {
case Oracle:
sqlStr = "SELECT COMMENTS FROM USER_TAB_COMMENTS WHERE TABLE_NAME= ? "; //$NON-NLS-1$
break;
case MySQL:
sqlStr = "SELECT TABLE_COMMENT FROM information_schema.TABLES WHERE TABLE_NAME= ? "; //$NON-NLS-1$
break;
default:
sqlStr = null;
}
return sqlStr;
}
/**
* get Comment Query Str.
*

View File

@@ -23,6 +23,7 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Stream;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
@@ -240,20 +241,14 @@ public class MigrationToolService implements IMigrationToolService {
sortMigrationTasks(toExecute);
final List<MigrationTask> done = new ArrayList<MigrationTask>(storedMigrations);
int nbMigrationsToDo = 0;
for (IProjectMigrationTask task : toExecute) {
MigrationTask mgTask = MigrationUtil.findMigrationTask(done, task);
if (mgTask == null && !task.isDeprecated()) {
nbMigrationsToDo++;
}
}
boolean hasTaskToExecute = toExecute.stream()
.anyMatch(task -> !task.isDeprecated() && MigrationUtil.findMigrationTask(done, task) == null);
// force to redo the migration task for the relations only if user ask for "clean" or if relations is empty
// or if there is at least another migration to do.
if (!beforeLogon
&& (!RelationshipItemBuilder.INDEX_VERSION.equals(project.getEmfProject().getItemsRelationVersion()) || nbMigrationsToDo > 0)) {
if (!beforeLogon && (!RelationshipItemBuilder.INDEX_VERSION.equals(project.getEmfProject().getItemsRelationVersion())
|| hasTaskToExecute)) {
// force to redo this migration task, to make sure the relationship is done correctly
// done.remove(RELATION_TASK);
MigrationUtil.removeMigrationTaskById(done, RELATION_TASK);
RelationshipItemBuilder.getInstance().unloadRelations();
@@ -264,41 +259,37 @@ public class MigrationToolService implements IMigrationToolService {
RelationshipItemBuilder.JOBLET_RELATION, true);
// reset
RelationshipItemBuilder.getInstance().unloadRelations();
nbMigrationsToDo++;
hasTaskToExecute = true;
}
if (nbMigrationsToDo == 0) {
boolean checkDupContext = !beforeLogon && Boolean.getBoolean("duplicate.context.reference.check");
if (!hasTaskToExecute && !checkDupContext) {
return;
}
// force execute migration in case user copy-past items with diffrent path on the file system and refresh
// the studio,it may cause bug TDI-19229
MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.FixProjectResourceLink");
// force to re-generate all job poms
MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.GenerateJobPomMigrationTask");
if (beforeLogon) {
// for every migration, force reset to default maven template
MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.ResetMavenTemplateMigrationTask");
if (checkDupContext) {
MigrationUtil.removeMigrationTaskById(done,
"org.talend.repository.model.migration.RemoveDuplicateContextReferencesMigrationTask");
}
if (hasTaskToExecute) {
// force execute migration in case user copy-past items with diffrent path on the file system and refresh
// the studio,it may cause bug TDI-19229
MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.FixProjectResourceLink");
// force to re-generate all job poms
MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.GenerateJobPomMigrationTask");
boolean haveAnyBinFolder = false; // to avoid some problems of migration, sometimes
for (ERepositoryObjectType type : (ERepositoryObjectType[]) ERepositoryObjectType.values()) {
if (!type.hasFolder()) {
continue;
if (beforeLogon) {
// for every migration, force reset to default maven template
MigrationUtil.removeMigrationTaskById(done,
"org.talend.repository.model.migration.ResetMavenTemplateMigrationTask");
}
String folderName = ERepositoryObjectType.getFolderName(type);
if (folderName == null || "".equals(folderName)) {
continue;
boolean hasBinFolder = Stream.of((ERepositoryObjectType[]) ERepositoryObjectType.values())
.filter(type -> type.hasFolder()).map(ERepositoryObjectType::getFolderName).filter(StringUtils::isNotBlank)
.map(folderName -> fsProject.getFolder(folderName))
.anyMatch(folder -> folder.exists() && folder.getFolder("bin").exists());
if (hasBinFolder) {
MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.RemoveBinFolderMigrationTask");
}
IFolder folder = fsProject.getFolder(folderName);
if (folder.exists() && folder.getFolder("bin").exists()) { //$NON-NLS-1$
haveAnyBinFolder = true;
break;
}
}
if (haveAnyBinFolder) {
MigrationUtil.removeMigrationTaskById(done, "org.talend.repository.model.migration.RemoveBinFolderMigrationTask");
}
final SubProgressMonitor subProgressMonitor = new SubProgressMonitor(monitorWrap, toExecute.size());

View File

@@ -44,8 +44,6 @@ import org.eclipse.jface.dialogs.MessageDialog;
import org.eclipse.jface.operation.IRunnableWithProgress;
import org.eclipse.jface.viewers.AbstractTreeViewer;
import org.eclipse.jface.viewers.ICheckStateProvider;
import org.eclipse.jface.viewers.ITreeViewerListener;
import org.eclipse.jface.viewers.TreeExpansionEvent;
import org.eclipse.jface.viewers.Viewer;
import org.eclipse.jface.viewers.ViewerFilter;
import org.eclipse.jface.wizard.WizardPage;
@@ -403,6 +401,9 @@ public class SelectorTableForm extends AbstractForm {
} else {
firstExpand = true;
}
if (treeItem.isDisposed()) {
return;
}
for (TreeItem item : treeItem.getItems()) {
if (item.getData() != null) {
TableNode node = (TableNode) item.getData();
@@ -507,140 +508,9 @@ public class SelectorTableForm extends AbstractForm {
return;
}
IMetadataConnection metadataConn = tableNode.getMetadataConn();
Connection conn = null;
Driver driver = null;
DatabaseMetaData dbMetaData = null;
ExtractMetaDataUtils extractMeta = ExtractMetaDataUtils.getInstance();
// Added by Marvin Wang on Mar. 13, 2013 for loading hive jars dynamically, refer to TDI-25072.
if (EDatabaseTypeName.HIVE.getXmlName().equalsIgnoreCase(metadataConn.getDbType())) {
try {
dbMetaData = HiveConnectionManager.getInstance().extractDatabaseMetaData(metadataConn);
} catch (Exception e) {
ExceptionHandler.process(e);
}
} else if (EDatabaseTypeName.IMPALA.getDisplayName().equalsIgnoreCase(metadataConn.getDbType())) {
try {
dbMetaData = ImpalaConnectionManager.getInstance().createConnection(metadataConn).getMetaData();
} catch (Exception e) {
ExceptionHandler.process(e);
}
} else {
List list = extractMeta.getConnectionList(metadataConn);
if (list != null && !list.isEmpty()) {
for (int i = 0; i < list.size(); i++) {
if (list.get(i) instanceof Connection) {
conn = (Connection) list.get(i);
}
if (list.get(i) instanceof DriverShim) {
driver = (DriverShim) list.get(i);
}
}
}
dbMetaData = extractMeta.getDatabaseMetaData(conn, metadataConn.getDbType(), metadataConn.isSqlMode(), metadataConn.getDatabase());
}
int type = tableNode.getType();
orgomg.cwm.objectmodel.core.Package pack = null;
List<MetadataTable> tableList = new ArrayList<MetadataTable>();
retrieveAllSubNodes(tableNode, tableList);
if (type == tableNode.CATALOG) {
if (tableNode.getChildren().isEmpty()) {
pack = tableNode.getCatalog();
}
} else if (type == tableNode.SCHEMA) {
pack = tableNode.getSchema();
}
try {
if (pack != null) {
TableInfoParameters paras = tableNode.getParas();
List<ETableTypes> paraType = paras.getTypes();
Set<String> availableTableTypes = new HashSet<String>();
for (ETableTypes tableType : paraType) {
availableTableTypes.add(tableType.getName());
}
// get all tables/views depending the filter selected
Set<String> tableNameFilter = null;
if (!paras.isUsedName()) {
tableNameFilter = new HashSet<String>();
if (paras.getSqlFiter() != null && !"".equals(paras.getSqlFiter())) { //$NON-NLS-1$
PreparedStatement stmt = extractMeta.getConn().prepareStatement(paras.getSqlFiter());
extractMeta.setQueryStatementTimeout(stmt);
ResultSet rsTables = stmt.executeQuery();
while (rsTables.next()) {
String nameKey = rsTables.getString(1).trim();
tableNameFilter.add(nameKey);
}
rsTables.close();
stmt.close();
}
} else {
tableNameFilter = paras.getNameFilters();
}
List<MetadataTable> tempListTables = new ArrayList<MetadataTable>();
MetadataFillFactory dbInstance = MetadataFillFactory.getDBInstance(metadataConn);
for (String filter : tableNameFilter) {
tempListTables = dbInstance.fillAll(pack, dbMetaData, metadataConn, null, filter, availableTableTypes.toArray(new String[] {}));
for (MetadataTable table : tempListTables) {
boolean contains = false;
for (MetadataTable inListTable : tableList) {
if (inListTable.getName().equals(table.getName())) {
contains = true;
break;
}
}
if (!contains) {
tableList.add(table);
}
}
}
if (tableNameFilter.isEmpty()) {
tempListTables = dbInstance.fillAll(pack, dbMetaData, metadataConn, null, null, availableTableTypes.toArray(new String[] {}));
for (MetadataTable table : tempListTables) {
boolean contains = false;
for (MetadataTable inListTable : tableList) {
if (inListTable.getName().equals(table.getName())) {
contains = true;
break;
}
}
if (!contains) {
tableList.add(table);
}
}
}
}
} catch (Exception e) {
ExceptionHandler.process(e);
} finally {
String dbType = metadataConn.getDbType();
// bug 22619
String driverClass = metadataConn.getDriverClass();
if (conn != null) {
ConnectionUtils.closeConnection(conn);
}
// for specific db such as derby
if (driver != null) {
if ((driverClass != null && driverClass.equals(EDatabase4DriverClassName.JAVADB_EMBEDED.getDriverClass()))
|| (dbType != null && (dbType.equals(EDatabaseTypeName.JAVADB_EMBEDED.getDisplayName()) || dbType.equals(EDatabaseTypeName.JAVADB_DERBYCLIENT.getDisplayName())
|| dbType.equals(EDatabaseTypeName.JAVADB_JCCJDBC.getDisplayName()) || dbType.equals(EDatabaseTypeName.HSQLDB_IN_PROGRESS.getDisplayName())))) {
try {
driver.connect("jdbc:derby:;shutdown=true", null); //$NON-NLS-1$
} catch (SQLException e) {
// exception of shutdown success. no need to catch.
}
}
}
}
transferToTableNode(tableList, tableNode);
Display.getDefault().syncExec(() -> {
viewer.setInput(tableNodeList);
viewer.expandToLevel(tableNode, viewer.ALL_LEVELS);
@@ -649,7 +519,7 @@ public class SelectorTableForm extends AbstractForm {
}
};
try {
this.parentWizardPage.getWizard().getContainer().run(true, true, runnable);
} catch (Exception e) {
@@ -657,6 +527,168 @@ public class SelectorTableForm extends AbstractForm {
}
}
private void retrieveAllSubNodes(TableNode tableNode, List<MetadataTable> tableList) {
tableList.clear();
List<TableNode> child = tableNode.getChildren();
boolean extended = false;
if (!child.isEmpty()) {
for (TableNode node : child) {
if (node.getType() == TableNode.TABLE) {
extended = true;
break;
}
}
}
// if extended is true, means table already got,no need to get again.
if (extended) {
return;
}
IMetadataConnection metadataConn = tableNode.getMetadataConn();
Connection conn = null;
Driver driver = null;
DatabaseMetaData dbMetaData = null;
ExtractMetaDataUtils extractMeta = ExtractMetaDataUtils.getInstance();
// Added by Marvin Wang on Mar. 13, 2013 for loading hive jars dynamically, refer to TDI-25072.
if (EDatabaseTypeName.HIVE.getXmlName().equalsIgnoreCase(metadataConn.getDbType())) {
try {
dbMetaData = HiveConnectionManager.getInstance().extractDatabaseMetaData(metadataConn);
} catch (Exception e) {
ExceptionHandler.process(e);
}
} else if (EDatabaseTypeName.IMPALA.getDisplayName().equalsIgnoreCase(metadataConn.getDbType())) {
try {
dbMetaData = ImpalaConnectionManager.getInstance().createConnection(metadataConn).getMetaData();
} catch (Exception e) {
ExceptionHandler.process(e);
}
} else {
List list = extractMeta.getConnectionList(metadataConn);
if (list != null && !list.isEmpty()) {
for (int i = 0; i < list.size(); i++) {
if (list.get(i) instanceof Connection) {
conn = (Connection) list.get(i);
}
if (list.get(i) instanceof DriverShim) {
driver = (DriverShim) list.get(i);
}
}
}
dbMetaData = extractMeta.getDatabaseMetaData(conn, metadataConn.getDbType(), metadataConn.isSqlMode(),
metadataConn.getDatabase());
}
int type = tableNode.getType();
orgomg.cwm.objectmodel.core.Package pack = null;
if (type == tableNode.CATALOG) {
if (tableNode.getChildren().isEmpty()) {
pack = tableNode.getCatalog();
} else {
for (TableNode n : tableNode.getChildren()) {
retrieveAllSubNodes(n, tableList);
}
}
} else if (type == tableNode.SCHEMA) {
pack = tableNode.getSchema();
}
try {
if (pack != null) {
TableInfoParameters paras = tableNode.getParas();
List<ETableTypes> paraType = paras.getTypes();
Set<String> availableTableTypes = new HashSet<String>();
for (ETableTypes tableType : paraType) {
availableTableTypes.add(tableType.getName());
}
// get all tables/views depending the filter selected
Set<String> tableNameFilter = null;
if (!paras.isUsedName()) {
tableNameFilter = new HashSet<String>();
if (paras.getSqlFiter() != null && !"".equals(paras.getSqlFiter())) { //$NON-NLS-1$
PreparedStatement stmt = extractMeta.getConn().prepareStatement(paras.getSqlFiter());
extractMeta.setQueryStatementTimeout(stmt);
ResultSet rsTables = stmt.executeQuery();
while (rsTables.next()) {
String nameKey = rsTables.getString(1).trim();
tableNameFilter.add(nameKey);
}
rsTables.close();
stmt.close();
}
} else {
tableNameFilter = paras.getNameFilters();
}
List<MetadataTable> tempListTables = new ArrayList<MetadataTable>();
MetadataFillFactory dbInstance = MetadataFillFactory.getDBInstance(metadataConn);
for (String filter : tableNameFilter) {
tempListTables = dbInstance.fillAll(pack, dbMetaData, metadataConn, null, filter,
availableTableTypes.toArray(new String[] {}));
for (MetadataTable table : tempListTables) {
boolean contains = false;
for (MetadataTable inListTable : tableList) {
if (inListTable.getName().equals(table.getName())) {
contains = true;
break;
}
}
if (!contains) {
tableList.add(table);
}
}
}
if (tableNameFilter.isEmpty()) {
tempListTables = dbInstance.fillAll(pack, dbMetaData, metadataConn, null, null,
availableTableTypes.toArray(new String[] {}));
for (MetadataTable table : tempListTables) {
boolean contains = false;
for (MetadataTable inListTable : tableList) {
if (inListTable.getName().equals(table.getName())) {
contains = true;
break;
}
}
if (!contains) {
tableList.add(table);
}
}
}
}
} catch (Exception e) {
ExceptionHandler.process(e);
} finally {
String dbType = metadataConn.getDbType();
// bug 22619
String driverClass = metadataConn.getDriverClass();
if (conn != null) {
ConnectionUtils.closeConnection(conn);
}
// for specific db such as derby
if (driver != null) {
if ((driverClass != null && driverClass.equals(EDatabase4DriverClassName.JAVADB_EMBEDED.getDriverClass()))
|| (dbType != null && (dbType.equals(EDatabaseTypeName.JAVADB_EMBEDED.getDisplayName())
|| dbType.equals(EDatabaseTypeName.JAVADB_DERBYCLIENT.getDisplayName())
|| dbType.equals(EDatabaseTypeName.JAVADB_JCCJDBC.getDisplayName())
|| dbType.equals(EDatabaseTypeName.HSQLDB_IN_PROGRESS.getDisplayName())))) {
try {
driver.connect("jdbc:derby:;shutdown=true", null); //$NON-NLS-1$
} catch (SQLException e) {
// exception of shutdown success. no need to catch.
}
}
}
}
if (!(tableNode.getType() == TableNode.CATALOG && pack == null)) {
transferToTableNode(tableList, tableNode);
}
}
protected void transferToTableNode(List<MetadataTable> list, TableNode parentNode) {
if (list != null && !list.isEmpty()) {
for (MetadataTable table : list) {
@@ -1149,6 +1181,10 @@ public class SelectorTableForm extends AbstractForm {
}
if (canAdd) {
tableNodes.add(schemaNode);
if (schemaNode.getValue() != null && StringUtils.isEmpty(schemaNode.getValue().trim())) {
List<MetadataTable> tableList = new ArrayList<MetadataTable>();
retrieveAllSubNodes(schemaNode, tableList);
}
}
}
}

View File

@@ -201,8 +201,8 @@ SHOW_IF="USE_FILE_AMBIGUOUS=='true'">
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Talen File Enhanced"
MODULE="talend_file_enhanced-1.1.jar"
MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" REQUIRED="true" />
MODULE="talend_file_enhanced-1.3.jar"
MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.3" REQUIRED="true" />
<IMPORT NAME="Talend_CSV" MODULE="talendcsv.jar"
MVN="mvn:org.talend.libraries/talendcsv/6.0.0"
REQUIRED="true" />