Compare commits

..

4 Commits

Author SHA1 Message Date
Dmytro Sylaiev
504dbef488 chore(TPS-5011): Bump the timestamp for tomorrow 2021-11-22 16:39:00 +02:00
Dmytro Sylaiev
41f35c4a21 chore(TPS-5011): Bump the release notes patch date 2021-11-22 11:58:04 +02:00
Dmytro Sylaiev
f46a9e3d62 patch(TPS-5011): Fix release notes 2021-11-19 11:07:08 +02:00
Dmytro Sylaiev
45f90f9d67 patch(TPS-5011): Apply the patch for patch branch 2021-11-18 18:43:43 +02:00
7 changed files with 79 additions and 187 deletions

View File

@@ -5,13 +5,13 @@ product:
- https://talend.poolparty.biz/coretaxonomy/23
---
# TPS-4436
# TPS-5011
| Info | Value |
| ---------------- | ---------------- |
| Patch Name | Patch\_20220309\_TPS-5130\_v1-7.2.1 |
| Release Date | 2022-03-09 |
| Target Version | 20190620\_1446-V7.2.1 |
| Patch Name | Patch\_20211123_TPS-5011\_v1-7.2.1 |
| Release Date | 2021-11-23 |
| Target Version | 20190620_1446-V7.2.1 |
| Product affected | Talend Studio |
## Introduction
@@ -24,8 +24,7 @@ This is a self-contained patch.
This patch contains the following fixes:
- [7.2.1] [Warn in CosmosDB]: Reduce to Trace "query does not contain any reference an index"
- [7.2.1] error in tCosmosdbInput component when Cosmos DB got upgraded from 3.2 to 3.6 version
- TPS-5011 [7.2.1]tMicrosoftCRMOutput - validate lookup update extremely slow in 721(TDI-44512)
## Prerequisites
@@ -57,11 +56,13 @@ Execute the following commands:
2. initRemote {tac_url} -ul {TAC login username} -up {TAC login password}
3. checkAndUpdate -tu {TAC login username} -tup {TAC login password}
## Affected files for this patch <!-- if applicable -->
## Uninstallation
Backup the Affected files list below. Uninstall the patch by restore the backup files.
## Affected files for this patch
The following files are installed by this patch:
- {Talend\_Studio\_path}/plugins/org.talend.designer.components.localprovider\_7.2.1.20190614\_0309/components/tFTPDelete/tFTPDelete\_main.javajet
- {Talend\_Studio\_path}/plugins/org.talend.designer.components.localprovider\_7.2.1.20190614\_0309/components/tFTPFileList/tFTPFileList\_begin.javajet
- {Talend\_Studio\_path}/plugins/org.talend.designer.components.localprovider\_7.2.1.20190614\_0309/components/tFTPGet/tFTPGet\_begin.javajet
- {Talend\_Studio\_path}/plugins/org.talend.designer.components.localprovider\_7.2.1.20190614\_0309/components/tFTPTruncate/tFTPTruncate\_main.javajet
- {Talend\_Studio\_path}/plugins/org.talend.designer.components.localprovider_7.2.1.20190614_0309/components/tMicrosoftCrmInput/tMicrosoftCrmInput_java.xml
- {Talend\_Studio\_path}/plugins/org.talend.designer.components.localprovider_7.2.1.20190614_0309/components/tMicrosoftCrmOutput/tMicrosoftCrmOutput_java.xml
- {Talend\_Studio\_path}/plugins/org.talend.libraries.crm_7.2.1.20181018_1137/lib/talend-mscrm-3.4-20200330.jar

View File

@@ -148,7 +148,7 @@
<%
}
%>
org.apache.commons.net.ftp.FTPFile[] filesInDirectory_<%=cid%> = null;
org.apache.commons.net.ftp.FTPFile[] filesInDirectory_<%=cid%> = null;
boolean getFileNamesFailed_<%=cid%> = false;
try {
filesInDirectory_<%=cid%> = ftp_<%=cid %>.listFiles();
@@ -163,24 +163,6 @@ org.apache.commons.net.ftp.FTPFile[] filesInDirectory_<%=cid%> = null;
%>
}
String[] listNamesInDirectory_<%=cid%> = null;
try {
listNamesInDirectory_<%=cid%> = ftp_<%=cid %>.listNames();
if(filesInDirectory_<%=cid%> == null){
getFileNamesFailed_<%=cid%> = true;
}
} catch(java.io.IOException e_<%=cid%>) {
globalMap.put("<%=cid%>_ERROR_MESSAGE",e_<%=cid%>.getMessage());
getFileNamesFailed_<%=cid%> = true;
<%
if(isLog4jEnabled) {
%>
log.warn("<%=cid%> - " + e_<%=cid%>.getMessage());
<%
}
%>
}
if (getFileNamesFailed_<%=cid %>) {
<%
if (isLog4jEnabled) {
@@ -195,42 +177,6 @@ org.apache.commons.net.ftp.FTPFile[] filesInDirectory_<%=cid%> = null;
%>
} else {
<% /*search files matches to filemask and delete them*/ %>
if( listNamesInDirectory_<%=cid%>!=null && (filesInDirectory_<%=cid%> == null || filesInDirectory_<%=cid%>.length == 0)){
for (String fileName_<%=cid%> : listNamesInDirectory_<%=cid%>) {
if (fileName_<%=cid%>.matches(mask<%=cid %>)) {
boolean successDeleted_<%=cid %> = false;
<%
if("FILE".equals(targetType)){
%>
successDeleted_<%=cid %> = ftp_<%=cid %>.deleteFile(fileName_<%=cid%>);
<%
}else{
%>
successDeleted_<%=cid %> = ftp_<%=cid %>.removeDirectory(fileName_<%=cid%>);
<%}%>
if (successDeleted_<%=cid %>) {
globalMap.put("<%=cid %>_CURRENT_STATUS", "File deleted.");
nb_file_<%=cid%>++;
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - <%=targetType%> '" + fileName_<%=cid%> + "' was deleted from the remote directory " + <%=remotedir %>);
<%}%>
} else {
String responceFTPMessage = ftp_<%=cid %>.getReplyString();
String errorMessageDeleteFail = "<%=cid%> - <%=targetType%> '" + fileName_<%=cid%> + "' was not deleted. Error message: " + responceFTPMessage;
<%if(isLog4jEnabled){%>
log.error(errorMessageDeleteFail);
<%} else { %>
System.err.println(errorMessageDeleteFail);
<%}%>
<%if (("true").equals(dieOnError)) { %>
throw new RuntimeException(errorMessageDeleteFail);
<% } %>
}
}
}
}else{
for (org.apache.commons.net.ftp.FTPFile file_<%=cid %> : filesInDirectory_<%=cid%>) {
String fileName_<%=cid%> = file_<%=cid %>.getName();
if (fileName_<%=cid%>.matches(mask<%=cid %>)) {
@@ -291,7 +237,6 @@ org.apache.commons.net.ftp.FTPFile[] filesInDirectory_<%=cid%> = null;
}
}
}
}// listNamesInDirectory_<%=cid%>!=null
}
<% /*return to root dir for next filemask*/ %>

View File

@@ -336,16 +336,7 @@ if (sftp) {// *** sftp *** //
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
}
ftpFiles_<%=cid %> = ftp_<%=cid %>.listFiles();
<%if(!dirFull){%>
String[] nameLists_<%=cid %> = ftp_<%=cid %>.listNames();
List<String> nameListsTemp_<%=cid %> = new java.util.ArrayList<>();
<%}%>
ftpFiles_<%=cid %> = ftp_<%=cid %>.listFiles();
<%
if ("true".equals(useExistingConn) && !moveToCurrentDir) {
%>
@@ -354,44 +345,35 @@ if (sftp) {// *** sftp *** //
<%
}
%>
<%/*prepare masks*/ %>
for (String filemask_<%=cid %> : maskList_<%=cid %>) {
java.util.regex.Pattern fileNamePattern_<%=cid %> = java.util.regex.Pattern.compile(filemask_<%=cid %>.replaceAll("\\.", "\\\\.").replaceAll("\\*", ".*"));
<%/*prepare masks*/ %>
for (String filemask_<%=cid %> : maskList_<%=cid %>) {
java.util.regex.Pattern fileNamePattern_<%=cid %> = java.util.regex.Pattern.compile(filemask_<%=cid %>.replaceAll("\\.", "\\\\.").replaceAll("\\*", ".*"));
<%if(!dirFull){%>
if(nameLists_<%=cid %> != null){
for (String ftpFile_<%=cid %> : nameLists_<%=cid %>) {
if (fileNamePattern_<%=cid %>.matcher(ftpFile_<%=cid %>).matches()) {
nameListsTemp_<%=cid %>.add(ftpFile_<%=cid %>);
}
}
}
<%}else{%>
for (org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> : ftpFiles_<%=cid %>) {
String ftpFileName_<%=cid %> = ftpFile_<%=cid %>.getName();
if (ftpFileName_<%=cid %>!=null && fileNamePattern_<%=cid %>.matcher(ftpFileName_<%=cid %>).matches()) {
fileListTemp_<%=cid %>.add(ftpFile_<%=cid %>);
}
}
<%}%>
}
for (org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> : ftpFiles_<%=cid %>) {
if (fileNamePattern_<%=cid %>.matcher(ftpFile_<%=cid %>.getName()).matches()) {
fileListTemp_<%=cid %>.add(ftpFile_<%=cid %>);
}
}
}
String currentFilePath_<%=cid%> = remotedir_<%=cid %>;
if(!remotedir_<%=cid%>.endsWith("/")&&!remotedir_<%=cid%>.endsWith("\\")){
currentFilePath_<%=cid%> += "/";
}
<%if(!dirFull){%>
for (String ftpFile_<%=cid %> : nameListsTemp_<%=cid %>) {
String currentFileName_<%=cid%> = ftpFile_<%=cid %>;
String currentFilePath_<%=cid%> = remotedir_<%=cid %>;
if(!remotedir_<%=cid%>.endsWith("/")&&!remotedir_<%=cid%>.endsWith("\\")){
currentFilePath_<%=cid%> += "/";
}
<%}else{%>
for (org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> : fileListTemp_<%=cid %>) {
String currentFileName_<%=cid%> = null;
currentFileName_<%=cid%> = ftpFile_<%=cid %>.toString();
for (int i = 0; i < fileListTemp_<%=cid %>.size(); i++) {
org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> = fileListTemp_<%=cid %>.get(i);
String currentFileName_<%=cid%> = null;
<%
}
if (dirFull) {
%>
currentFileName_<%=cid%> = ftpFile_<%=cid %>.toString();
<%
} else {
%>
currentFileName_<%=cid%> = ftpFile_<%=cid %>.getName();
<%
}
if(isLog4jEnabled) {
%>
@@ -399,17 +381,10 @@ if (sftp) {// *** sftp *** //
<%
}
%>
<%/*set to globalMap*/ %>
globalMap.put("<%=cid %>_CURRENT_FILE", currentFileName_<%=cid%>);
<%if(!dirFull){%>
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%> + currentFileName_<%=cid%>);
<%}else{%>
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%> + ftpFile_<%=cid %>.getName());
<%}%>
nb_file_<%=cid %>++;
<%/*set to globalMap*/ %>
globalMap.put("<%=cid %>_CURRENT_FILE", currentFileName_<%=cid%>);
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%> + ftpFile_<%=cid %>.getName());
nb_file_<%=cid %>++;
<%
} else { // *** ftp *** //
%>

View File

@@ -584,60 +584,38 @@ if (sftp) { // *** sftp *** //
String path = ftpClient.printWorkingDirectory();
org.apache.commons.net.ftp.FTPFile[] ftpFiles = null;
ftpFiles = ftpClient.listFiles();
String[] nameLists = null;
try {
nameLists = ftpClient.listNames();
} catch (IOException e) {
e.printStackTrace();
}
if( nameLists!=null && (ftpFiles == null || ftpFiles.length == 0)){
//if the file is folder, catch the FTPException and recur
for (String ftpFileName : nameLists){
try{
downloadFile(localDirectory + "/" + ftpFileName,ftpFileName);
} catch (IOException e) {
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - " + e.getMessage());
<%}%>
java.io.File localFile = new java.io.File(localDirectory + "/" + ftpFileName);
if (!localFile.exists()) {
localFile.mkdir();
}
getAllFiles(path + "/" + ftpFileName, localDirectory + "/" + ftpFileName);
chdir(remoteDirectory);
}
}
}else{
for (org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
for (org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
if (ftpFile.isDirectory()) {
if (ftpFile.isDirectory()) {
if ((!(".").equals(ftpFile.getName())) && (!("..").equals(ftpFile.getName()))) {
java.io.File localFile = new java.io.File(localDirectory + "/" + ftpFile.getName());
if ((!(".").equals(ftpFile.getName())) && (!("..").equals(ftpFile.getName()))) {
java.io.File localFile = new java.io.File(localDirectory + "/" + ftpFile.getName());
if (!localFile.exists()) {
localFile.mkdir();
}
getAllFiles(path + "/" + ftpFile.getName(), localDirectory + "/" + ftpFile.getName());
chdir(path);
}
} else if (!ftpFile.isSymbolicLink()) {
downloadFile(localDirectory + "/" + ftpFile.getName(),ftpFile.getName());
}
}
if (!localFile.exists()) {
localFile.mkdir();
}
getAllFiles(path + "/" + ftpFile.getName(), localDirectory + "/" + ftpFile.getName());
chdir(path);
}
} else if (!ftpFile.isSymbolicLink()) {
downloadFile(localDirectory + "/" + ftpFile.getName(),ftpFile.getName());
}
}
}
public void getFiles(String remoteDirectory, String localDirectory, String maskStr)
throws IllegalStateException, IOException, java.io.FileNotFoundException {
chdir(remoteDirectory);
String[] nameLists = ftpClient.listNames("");
if(nameLists != null){
for (String fileName : nameLists) {
org.apache.commons.net.ftp.FTPFile[] ftpFiles = null;
ftpFiles = ftpClient.listFiles(".");
for(org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
if (ftpFile.isFile()) {
String fileName = ftpFile.getName();
if (fileName.matches(maskStr)) {
downloadFile(localDirectory + "/" + fileName,fileName);
}
@@ -645,12 +623,12 @@ if (sftp) { // *** sftp *** //
}
}
public boolean chdir(String path) throws IllegalStateException, IOException {
if (!".".equals(path)) {
return ftpClient.changeWorkingDirectory(path);
}
return true;
}
public void chdir(String path)
throws IllegalStateException, IOException {
if (!".".equals(path)) {
ftpClient.changeWorkingDirectory(path);
}
}
public String pwd()
throws IllegalStateException, IOException {
@@ -660,15 +638,11 @@ if (sftp) { // *** sftp *** //
private void downloadFile(String localFileName, String remoteFileName)
throws IllegalStateException, java.io.FileNotFoundException, IOException {
try (java.io.FileOutputStream localFos = new java.io.FileOutputStream(localFileName)) {
boolean status = ftpClient.retrieveFile(remoteFileName, localFos);
if(status){
ftpClient.retrieveFile(remoteFileName, localFos);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
<%}%>
count++;
}else{
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer fail.");
}
}
}
}

View File

@@ -128,15 +128,11 @@
<%
}
%>
String[] filesInDirectory_<%=cid%> = null;
org.apache.commons.net.ftp.FTPFile[] filesInDirectory_<%=cid%> = null;
boolean getFileNamesFailed_<%=cid%> = false;
try {
filesInDirectory_<%=cid%> = ftp_<%=cid %>.listNames();
if(filesInDirectory_<%=cid%> == null){
getFileNamesFailed_<%=cid%> = true;
}
filesInDirectory_<%=cid%> = ftp_<%=cid %>.listFiles();
} catch(java.io.IOException e_<%=cid%>) {
globalMap.put("<%=cid%>_ERROR_MESSAGE",e_<%=cid%>.getMessage());
getFileNamesFailed_<%=cid%> = true;
<%
if(isLog4jEnabled) {
@@ -161,10 +157,11 @@
%>
} else {
<% /*search files matches to filemask and truncate them*/ %>
for (String fileName_<%=cid %> : filesInDirectory_<%=cid%>) {
for (org.apache.commons.net.ftp.FTPFile file_<%=cid %> : filesInDirectory_<%=cid%>) {
String fileName_<%=cid%> = file_<%=cid %>.getName();
if (fileName_<%=cid%>.matches(mask<%=cid %>)) {
boolean successTruncated_<%=cid %> = false;
if (file_<%=cid %>.isFile()) {
successTruncated_<%=cid %> = ftp_<%=cid %>.storeFile(fileName_<%=cid%>, new java.io.ByteArrayInputStream(new byte[]{}));
if (successTruncated_<%=cid %>) {
globalMap.put("<%=cid %>_CURRENT_STATUS", "File truncate OK.");
@@ -183,7 +180,7 @@
globalMap.put("<%=cid %>_CURRENT_STATUS", "File truncate fail.");
throw new RuntimeException(errorMessageDeleteFail);
}
}
}
}
}

View File

@@ -27523,7 +27523,7 @@
<IMPORT NAME="jcifs" MODULE="jcifs-1.3.0.jar" MVN="mvn:org.talend.libraries/jcifs-1.3.0/6.0.0" REQUIRED_IF="((AUTH_TYPE == 'ON_PREMISE') AND (MS_CRM_VERSION == 'CRM_2011')) OR (API_VERSION=='API_2007')" />
<!-- 2011 -->
<!-- crm client -->
<IMPORT NAME="talend-mscrm-3.4-20190513" MODULE="talend-mscrm-3.4-20190513.jar" MVN="mvn:org.talend.libraries/talend-mscrm-3.4-20190513/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.crm/lib/talend-mscrm-3.4-20190513.jar" REQUIRED_IF="(((AUTH_TYPE=='ONLINE') AND (API_VERSION=='API_2011' OR API_VERSION =='API_2016_ODATA' OR API_VERSION =='API_2018_ODATA')) OR ((AUTH_TYPE=='ON_PREMISE') AND (MS_CRM_VERSION == 'CRM_2016' OR MS_CRM_VERSION == 'CRM_2018')))" />
<IMPORT NAME="talend-mscrm-3.4-20200330" MODULE="talend-mscrm-3.4-20200330.jar" MVN="mvn:org.talend.libraries/talend-mscrm-3.4-20200330/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.crm/lib/talend-mscrm-3.4-20200330.jar" REQUIRED_IF="(((AUTH_TYPE=='ONLINE') AND (API_VERSION=='API_2011' OR API_VERSION =='API_2016_ODATA' OR API_VERSION =='API_2018_ODATA')) OR ((AUTH_TYPE=='ON_PREMISE') AND (MS_CRM_VERSION == 'CRM_2016' OR MS_CRM_VERSION == 'CRM_2018')))" />
<!-- axis2 1.7.4 -->
<IMPORT NAME="activation-1.1" MODULE="activation-1.1.jar" MVN="mvn:org.talend.libraries/activation-1.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.axis2/lib/activation-1.1.jar" REQUIRED_IF="(AUTH_TYPE=='ONLINE') AND (API_VERSION=='API_2011')" />
<IMPORT NAME="axiom-api-1.2.20" MODULE="axiom-api-1.2.20.jar" MVN="mvn:org.talend.libraries/axiom-api-1.2.20/6.0.0" REQUIRED_IF="(AUTH_TYPE=='ONLINE') AND (API_VERSION=='API_2011')" />

View File

@@ -38286,7 +38286,7 @@
<IMPORT NAME="jcifs" MODULE="jcifs-1.3.0.jar" MVN="mvn:org.talend.libraries/jcifs-1.3.0/6.0.0" REQUIRED_IF="((AUTH_TYPE == 'ON_PREMISE') AND (MS_CRM_VERSION == 'CRM_2011')) OR (API_VERSION=='API_2007')" />
<!-- 2011 -->
<!-- crm client -->
<IMPORT NAME="talend-mscrm-3.4-20190513" MODULE="talend-mscrm-3.4-20190513.jar" MVN="mvn:org.talend.libraries/talend-mscrm-3.4-20190513/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.crm/lib/talend-mscrm-3.4-20190513.jar" REQUIRED_IF="((AUTH_TYPE=='ONLINE') AND (API_VERSION=='API_2011' OR API_VERSION =='API_2016_ODATA' OR API_VERSION =='API_2018_ODATA')) OR ((AUTH_TYPE == 'ON_PREMISE') AND (MS_CRM_VERSION == 'CRM_2016' OR MS_CRM_VERSION == 'CRM_2018'))" />
<IMPORT NAME="talend-mscrm-3.4-20200330" MODULE="talend-mscrm-3.4-20200330.jar" MVN="mvn:org.talend.libraries/talend-mscrm-3.4-20200330/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.crm/lib/talend-mscrm-3.4-20200330.jar" REQUIRED_IF="((AUTH_TYPE=='ONLINE') AND (API_VERSION=='API_2011' OR API_VERSION =='API_2016_ODATA' OR API_VERSION =='API_2018_ODATA')) OR ((AUTH_TYPE == 'ON_PREMISE') AND (MS_CRM_VERSION == 'CRM_2016' OR MS_CRM_VERSION == 'CRM_2018'))" />
<!-- axis2 1.7.4 -->
<IMPORT NAME="activation-1.1" MODULE="activation-1.1.jar" MVN="mvn:org.talend.libraries/activation-1.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.axis2/lib/activation-1.1.jar" REQUIRED_IF="(AUTH_TYPE=='ONLINE') AND (API_VERSION=='API_2011')" />
<IMPORT NAME="axiom-api-1.2.20" MODULE="axiom-api-1.2.20.jar" MVN="mvn:org.talend.libraries/axiom-api-1.2.20/6.0.0" REQUIRED_IF="(AUTH_TYPE=='ONLINE') AND (API_VERSION=='API_2011')" />