Compare commits

..

2 Commits

Author SHA1 Message Date
Christophe Le Saec
20bf94ec21 modif .md 2019-09-06 11:54:26 +02:00
Christophe Le Saec
fd60c1edca patch notes 2019-09-05 15:45:16 +02:00
324 changed files with 4053 additions and 8259 deletions

View File

@@ -3,15 +3,17 @@ version: 7.2.1
module: https://talend.poolparty.biz/coretaxonomy/42
product:
- https://talend.poolparty.biz/coretaxonomy/23
- https://talend.poolparty.biz/coretaxonomy/26
- https://talend.poolparty.biz/coretaxonomy/24
---
# TPS-4808
# TPS-3403
| Info | Value |
| ---------------- | ---------------- |
| Patch Name | Patch\_20210521\_TPS-4808\_v1-7.2.1 |
| Release Date | 2021-05-21 |
| Target Version | 20190620_1446-V7.2.1 |
| Patch Name | Patch\_20190907\_TPS-3403\_v1-7.2.1 |
| Release Date | 2019-09-07 |
| Target Version | 20190620\_1446-V7.2.1 |
| Product affected | Talend Studio |
## Introduction
@@ -21,24 +23,21 @@ This is a self-contained patch.
**NOTE**: For information on how to obtain this patch, reach out to your Support contact at Talend.
## Fixed issues
This patch contains this following fixe:
This patch contains the following fixes:
- TPS-4808 [7.2.1] tSoap components doesn't work and returning a 400 Bad request message(TDI-44893)
- TPS-3403 filecopy fail with java11 (TDI-42230)
## Prerequisites
Consider the following requirements for your system:
- Talend Studio 7.2.1 must be installed.
## Installation
**NOTE**: If the patch is deployed in the approach Installing the patch using Talend Studio, the folder configuration under this patch must be replaced manually.
## Installation
### Installing the patch using Software update
1) Logon TAC and switch to Configuration->Software Update, then enter the correct values and save referring to the documentation: https://help.talend.com/reader/f7Em9WV_cPm2RRywucSN0Q/j9x5iXV~vyxMlUafnDejaQ
1) Logon TAC and switch to Configuration->Software Update, then enter the correct values and save referring to the documentation: https://help.talend.com/reader/f7Em9WV\_cPm2RRywucSN0Q/j9x5iXV~vyxMlUafnDejaQ
2) Switch to Software update page, where the new patch will be listed. The patch can be downloaded from here into the nexus repository.
@@ -54,6 +53,14 @@ Consider the following requirements for your system:
Execute the following commands:
1. Talend-Studio-win-x86_64.exe -nosplash -application org.talend.commandline.CommandLine -consoleLog -data commandline-workspace startServer -p 8002 --talendDebug
2. initRemote {tac_url} -ul {TAC login username} -up {TAC login password}
1. Talend-Studio-win-x86\_64.exe -nosplash -application org.talend.commandline.CommandLine -consoleLog -data commandline-workspace startServer -p 8002 --talendDebug
2. initRemote {tac\_url} -ul {TAC login username} -up {TAC login password}
3. checkAndUpdate -tu {TAC login username} -tup {TAC login password}
## Affected files for this patch
The following files are installed by this patch:
- plugins/org.talend.designer.components.localprovider\_7.2.1.20190614\_0309/components/tFileCopy/tFileCopy\_java.xml
- plugins/org.talend.libraries.custom\_7.2.1.20181114\_0551/lib/filecopy-2.0.0.jar

View File

@@ -106,21 +106,6 @@
exist_tSQLDB = true;
}
String[] tcompV0Components = new String[] {
"tAzureStorageConnection",
"tGoogleDriveConnection",
"tMarketoConnection",
"tSnowflakeConnection",
"tSalesforceConnection",
"tNetsuiteConnection",
"tMarkLogicConnection"
};
List<INode> tcompV0ComponentsList = new ArrayList<INode>();
for (String tcompV0Component : tcompV0Components) {
tcompV0ComponentsList.addAll((List<INode>)process.getNodesOfType(tcompV0Component));
}
boolean exist_tSCP = false;
List<INode> scpComponentsList = (List<INode>)process.getNodesOfType("tSCPConnection");
if (scpComponentsList.size() > 0) {
@@ -180,8 +165,6 @@
exist_tSalesforce = true;
}
List<INode> snowflakeComponentsList = (List<INode>)process.getNodesOfType("tSnowflakeConnection");
boolean exist_tSAP= false;
List<INode> sapComponentsList = (List<INode>)process.getNodesOfType("tSAPConnection");
if (sapComponentsList.size() > 0) {
@@ -462,7 +445,6 @@
lastStr = "";
}
}
enableLogStash = "true".equalsIgnoreCase(System.getProperty("monitoring"));
<%if(isLog4jEnabled){%>
if(!"".equals(log4jLevel)){
@@ -511,13 +493,7 @@
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("appender.file.maxsize", "52428800");
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("appender.file.maxbackup", "20");
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("host", "false");
System.getProperties().stringPropertyNames().stream()
.filter(it -> it.startsWith("monitoring.audit.logger.properties."))
.forEach(key -> properties_<%=jobCatcherNode.getUniqueName()%>.setProperty(key.substring("monitoring.audit.logger.properties.".length()), System.getProperty(key)));
org.apache.log4j.Logger.getLogger("audit").setLevel(org.apache.log4j.Level.DEBUG);
auditLogger_<%=jobCatcherNode.getUniqueName()%> = org.talend.job.audit.JobEventAuditLoggerFactory.createJobAuditLogger(properties_<%=jobCatcherNode.getUniqueName()%>);
}
<%
@@ -608,16 +584,7 @@
} //endIf
%>
context.setContextType("<%=ctxParam.getName()%>", "<%=ctxParam.getType()%>");
if(context.getStringValue("<%=ctxParam.getName()%>") == null) {
context.<%=ctxParam.getName()%> = null;
} else {
<%//}B1 block%>
<%
String typeToGenerate ="String";
if( !(ctxParam.getType().equals("id_File") || ctxParam.getType().equals("id_Directory") ||ctxParam.getType().equals("id_List Of Value"))) {
typeToGenerate=JavaTypesManager.getTypeToGenerate(ctxParam.getType(),true);
}
if (ctxParam.getType().equals("id_Password")) {
%>
String pwd_<%=ctxParam.getName()%>_value = context.getProperty("<%=ctxParam.getName()%>");
@@ -635,7 +602,13 @@
}
}
<%
} else if(typeToGenerate.equals("java.util.Date")) {
continue;
}
String typeToGenerate ="String";
if( !(ctxParam.getType().equals("id_File") || ctxParam.getType().equals("id_Directory") ||ctxParam.getType().equals("id_List Of Value"))) {
typeToGenerate=JavaTypesManager.getTypeToGenerate(ctxParam.getType(),true);
}
if(typeToGenerate.equals("java.util.Date")) {
%>
try{
String context_<%=ctxParam.getName()%>_value = context.getProperty("<%=ctxParam.getName()%>");
@@ -694,10 +667,6 @@
}
<%
}
%>
<%//{B1 block%>
}
<%
}
%>
} <%/*close last method*/%>
@@ -1290,11 +1259,13 @@ if (execStat) {
} else {
type = "ftp";
}
if ("sftp".equals(type)) { %>
if ("ftp".equals(type)) { %>
((com.enterprisedt.net.ftp.FTPClient) obj_conn).quit();
<%} else if ("sftp".equals(type)) { %>
((com.jcraft.jsch.ChannelSftp) obj_conn).quit();
<%} else { %>
((org.apache.commons.net.ftp.FTPClient) obj_conn).disconnect();
<%}%>
<%} else { %>
((org.apache.commons.net.ftp.FTPSClient) obj_conn).disconnect();
<%}%>
}
<% } %>
} catch (java.lang.Exception e) {
@@ -1469,14 +1440,12 @@ if (execStat) {
<% } %>
<% for (INode existNode : existComponentsList) { %>
connections.put("col_<%=existNode.getUniqueName()%>", globalMap.get("col_<%=existNode.getUniqueName()%>"));
<% }
<% } %>
for (INode tcompV0ConnectionNode : tcompV0ComponentsList) { %>
connections.put("<%=tcompV0ConnectionNode.getUniqueName()%>_connection", globalMap.get("<%=tcompV0ConnectionNode.getUniqueName()%>_connection"));
connections.put("<%=tcompV0ConnectionNode.getUniqueName()%>_COMPONENT_RUNTIME_PROPERTIES", globalMap.get("<%=tcompV0ConnectionNode.getUniqueName()%>_COMPONENT_RUNTIME_PROPERTIES"));
<% }
for (INode sapNode : sapComponentsList) { %>
<% for (INode salesforceNode : salesforceComponentsList) { %>
connections.put("conn_<%=salesforceNode.getUniqueName()%>", globalMap.get("conn_<%=salesforceNode.getUniqueName()%>"));
<% } %>
<% for (INode sapNode : sapComponentsList) { %>
connections.put("conn_<%=sapNode.getUniqueName()%>", globalMap.get("conn_<%=sapNode.getUniqueName()%>"));
<% } %>
<% for (INode gsNode : gsComponentsList) { %>
@@ -1540,10 +1509,12 @@ if (execStat) {
}
} else if (arg.startsWith("--log4jLevel=")) {
log4jLevel = arg.substring(13);
} else if (arg.startsWith("--monitoring") && arg.contains("=")) {//for trunjob call
final int equal = arg.indexOf('=');
final String key = arg.substring("--".length(), equal);
System.setProperty(key, arg.substring(equal + 1));
} else if (arg.startsWith("--monitoring=")) {//for trunjob call
enableLogStash = "true".equalsIgnoreCase(arg.substring(13));
}
if(!enableLogStash) {
enableLogStash = "true".equalsIgnoreCase(System.getProperty("monitoring"));
}
}

View File

@@ -62,8 +62,6 @@ class IndexedRecordToRowStructGenerator {
%>
boolean <%=codeVarIsDynamicInitialized%> = false;
routines.system.Dynamic <%=codeVarDynamic%> = new routines.system.Dynamic();
//Workaround for TDI-44051, TcimpV0 do not need DbmsID
<%=codeVarDynamic%>.setDbmsId("<%=cid%>");
<%
}
@@ -117,11 +115,10 @@ class IndexedRecordToRowStructGenerator {
for (org.apache.avro.Schema.Field dynamicField_<%=cid%> : dynSchema_<%=cid%>.getFields()){
routines.system.DynamicMetadata dynamicMetadata_<%=cid%> = new routines.system.DynamicMetadata();
org.apache.avro.Schema dynamicFieldSchema_<%=cid%> = dynamicField_<%=cid%>.schema();
String dbName = dynamicField_<%=cid%>.getProp("talend.field.dbColumnName");
// set name
dynamicMetadata_<%=cid%>.setName(dynamicField_<%=cid%>.name());
// set db name
dynamicMetadata_<%=cid%>.setDbName(dbName==null?dynamicField_<%=cid%>.name():dbName);
dynamicMetadata_<%=cid%>.setDbName(dynamicField_<%=cid%>.name());
// set nullable
if (org.talend.daikon.avro.AvroUtils.isNullable(dynamicFieldSchema_<%=cid%>)) {
dynamicMetadata_<%=cid%>.setNullable(true);

View File

@@ -289,15 +289,6 @@ if(isRunInMultiThread){
}
<% } %>
}
//if the stored or passed value is "<TALEND_NULL>" string, it mean null
public String getStringValue(String key) {
String origin_value = this.getProperty(key);
if(NULL_VALUE_EXPRESSION_IN_COMMAND_STRING_FOR_CHILD_JOB_ONLY.equals(origin_value)) {
return null;
}
return origin_value;
}
<%
for (IContextParameter ctxParam :params)

View File

@@ -66,7 +66,6 @@ import org.talend.designer.codegen.model.CodeGeneratorEmittersPoolFactory;
import org.talend.designer.codegen.model.CodeGeneratorInternalTemplatesFactoryProvider;
import org.talend.designer.codegen.proxy.JetProxy;
import org.talend.designer.core.generic.model.Component;
import org.talend.designer.runprocess.ProcessorUtilities;
/**
* CodeGenerator.
@@ -354,12 +353,8 @@ public class CodeGenerator implements ICodeGenerator {
.getDefault().getService(IResourcesDependenciesService.class);
String resourcePathForContext = null;
if (process instanceof IProcess2) {
String value = contextPar.getValue();
// for runtime
if (!ProcessorUtilities.isExportConfig()) {
contextPar.setType(JavaTypesManager.FILE.getId());
}
resourcePathForContext = resourceService.getResourcePathForContext(process, value);
resourcePathForContext = resourceService.getResourcePathForContext(process,
contextPar.getValue());
} else {
// for PreviewFileInputContentDataProcess run
resourcePathForContext = resourceService.getResourceItemFilePath(contextPar.getValue());

View File

@@ -94,7 +94,7 @@ public class JavaRoutineSynchronizer extends AbstractRoutineSynchronizer {
private void syncRoutineItems(Collection<RoutineItem> routineObjects, boolean forceUpdate) throws SystemException {
for (RoutineItem routineItem : routineObjects) {
syncRoutine(routineItem, true, forceUpdate);
syncRoutine(routineItem, true, true, forceUpdate);
}
syncSystemRoutine(ProjectManager.getInstance().getCurrentProject());
}

View File

@@ -28,9 +28,9 @@ import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.ResourceBundle;
import java.util.Set;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
@@ -70,6 +70,7 @@ import org.talend.core.model.components.ComponentCategory;
import org.talend.core.model.components.ComponentManager;
import org.talend.core.model.components.ComponentProviderInfo;
import org.talend.core.model.components.ComponentUtilities;
import org.talend.core.model.components.EComponentType;
import org.talend.core.model.components.IComponent;
import org.talend.core.model.components.IComponentsFactory;
import org.talend.core.model.components.IComponentsHandler;
@@ -837,21 +838,23 @@ public class ComponentsFactory implements IComponentsFactory {
init(false);
}
// check if reference joblet component presents
JobletUtil jobletUtils = new JobletUtil();
Optional<IComponent> result = jobletUtils.findComponentByName(componentList, name, paletteType);
if (!result.isPresent()) {
// check if any name matching joblet component presents
if (jobletUtils.matchExpression(name)) {
name = StringUtils.substringAfterLast(name, ":"); //$NON-NLS-1$
if (StringUtils.isNotBlank(name)) {
result = jobletUtils.findComponentByName(componentList, name, paletteType);
for (IComponent comp : componentList) {
if (comp.getComponentType() != EComponentType.JOBLET) {
continue;
}
String comName = comp.getName();
if (comp != null && paletteType.equals(comp.getPaletteType())) {
if (comName.equals(name)) {
return comp;
} else if (new JobletUtil().matchExpression(comName)) {
String[] names = comName.split(":"); //$NON-NLS-1$
comName = names[1];
if (comName.equals(name)) {
return comp;
}
}
}
}
if (result.isPresent()) {
return result.get();
}
return null;
}

View File

@@ -41,8 +41,8 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.19</version>
</dependency>
<version>1.10</version>
</dependency>
</dependencies>
<build>
<resources>

View File

@@ -7,7 +7,7 @@
<groupId>net.sf.json-lib</groupId>
<artifactId>json-lib</artifactId>
<packaging>jar</packaging>
<version>2.4.5-talend</version>
<version>2.4.2-talend</version>
<name>json-lib</name>
<properties>

View File

@@ -104,17 +104,16 @@ public final class JSONUtils {
return "null";
}
// Shave off trailing zeros. Keep decimal to keep type double
// Shave off trailing zeros and decimal point, if possible.
String s = Double.toString( d );
if( s.indexOf( '.' ) > 0 && s.indexOf( 'e' ) < 0 && s.indexOf( 'E' ) < 0 ){
while(s.charAt(s.length() - 1) == '0'){
if(s.endsWith(".0")){
break;
}
while( s.endsWith( "0" ) ){
s = s.substring( 0, s.length() - 1 );
}
}
if( s.endsWith( "." ) ){
s = s.substring( 0, s.length() - 1 );
}
}
return s;
}
@@ -441,19 +440,17 @@ public final class JSONUtils {
}
testValidity( n );
// Shave off trailing zeros. Keep decimal to keep type double
// Shave off trailing zeros and decimal point, if possible.
String s = n.toString();
if( s.indexOf( '.' ) > 0 && s.indexOf( 'e' ) < 0 && s.indexOf( 'E' ) < 0 ){
while(s.charAt(s.length() - 1) == '0'){
if(s.endsWith(".0")){
break;
}
while( s.endsWith( "0" ) ){
s = s.substring( 0, s.length() - 1 );
}
if( s.endsWith( "." ) ){
s = s.substring( 0, s.length() - 1 );
}
}
return s;
}

View File

@@ -163,11 +163,6 @@ public class XMLSerializer {
*/
private boolean useLongDecimals;
/**
* flag for if parse empty elements as empty strings
*/
private boolean useEmptyStrings;
/**
* Creates a new XMLSerializer with default options.<br>
* <ul>
@@ -808,8 +803,6 @@ public class XMLSerializer {
clazz = JSONTypes.OBJECT;
}else if( JSONTypes.ARRAY.compareToIgnoreCase( clazzText ) == 0 ){
clazz = JSONTypes.ARRAY;
} else if(JSONTypes.STRING.equalsIgnoreCase(clazzText)) {
clazz = JSONTypes.STRING;
}
}
return clazz;
@@ -1421,8 +1414,6 @@ public class XMLSerializer {
String text = element.getValue();
params = StringUtils.split( paramsAttribute.getValue(), "," );
setOrAccumulate( jsonObject, key, new JSONFunction( params, text ) );
} else if( useEmptyStrings && clazz != null && clazz.equalsIgnoreCase(JSONTypes.STRING) ) {
setTextValue(jsonObject, key, element);
}else{
if( isArray( element, false ) ){
setOrAccumulate( jsonObject, key, processArrayElement( element, defaultType ) );
@@ -1430,23 +1421,19 @@ public class XMLSerializer {
setOrAccumulate( jsonObject, key, simplifyValue( jsonObject,
processObjectElement( element, defaultType ) ) );
}else{
setTextValue(jsonObject, key, element);
String value;
if( isKeepCData && isCData( element ) ){
value = "<![CDATA[" + element.getValue() + "]]>";
}else{
value = element.getValue();
}
setOrAccumulate( jsonObject, key, trimSpaceFromValue( value ) );
}
}
}
}
}
private void setTextValue(final JSONObject jsonObject, final String key, final Element element) {
String value;
if( isKeepCData && isCData( element ) ){
value = "<![CDATA[" + element.getValue() + "]]>";
}else{
value = element.getValue();
}
setOrAccumulate( jsonObject, key, trimSpaceFromValue( value ) );
}
private boolean isCData( Element element ) {
if( element.getChildCount() == 1 ){
final Node child = element.getChild( 0 );
@@ -1506,14 +1493,6 @@ public class XMLSerializer {
return str;
}
public void setUseEmptyStrings(boolean useEmptyStrings) {
this.useEmptyStrings = useEmptyStrings;
}
public boolean isUseEmptyStrings() {
return this.useEmptyStrings;
}
private static class CustomElement extends Element {
private static String getName( String name ) {
int colon = name.indexOf( ':' );

View File

@@ -44,25 +44,7 @@ public class TestJSONUtils extends TestCase {
}
public void testDoubleToString_trailingZeros() {
assertEquals( "200.0", JSONUtils.doubleToString( 200.00000 ) );
}
public void testDoubleToString() {
Map<String, Double> expected = new HashMap<>();
expected.put("200.0", 200.0d);
expected.put("200.0", 200.000d);
expected.put("200.1", 200.1d);
expected.put("200.1", 200.10d);
expected.put("200.1", 200.1000d);
expected.put("200.12345", 200.12345d);
expected.put("200.12345", 200.123450000d);
expected.put("200.101", 200.101d);
expected.put("1.0E-8", 1.0E-8);
expected.put("200.0", 200d);
for(String key : expected.keySet()){
assertEquals(key,JSONUtils.doubleToString(expected.get(key)));
}
assertEquals( "200", JSONUtils.doubleToString( 200.00000 ) );
}
public void testGetFunctionParams() {
@@ -119,28 +101,6 @@ public class TestJSONUtils extends TestCase {
}
}
public void testNumberToString() {
Map<String, Number> expected = new HashMap<>();
expected.put("0", Integer.valueOf("00000"));
expected.put("123", Integer.valueOf("123"));
expected.put("-123", Integer.valueOf("-123"));
expected.put("0.0", Double.valueOf("0"));
expected.put("0.0", Double.valueOf("0.00000"));
expected.put("120.0001", Double.valueOf("120.0001000"));
expected.put("-120.0001", Double.valueOf("-120.0001000"));
expected.put("0.0", Float.valueOf("0"));
expected.put("0.0", Float.valueOf("0.00000"));
expected.put("120.0001", Float.valueOf("120.0001000"));
expected.put("-120.0001", Float.valueOf("-120.0001000"));
for(String key : expected.keySet()) {
assertEquals(key, JSONUtils.numberToString(expected.get(key)));
}
}
public void testQuote_emptyString() {
assertEquals( "\"\"", JSONUtils.quote( "" ) );
}

View File

@@ -1,17 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.talend.libraries</groupId>
<artifactId>job-audit</artifactId>
<version>1.0.1</version>
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.talend.libraries</groupId>
<artifactId>job-audit</artifactId>
<version>1.0</version>
<properties>
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
</properties>
<repositories>
<repository>
<id>talend_nexus</id>
@@ -23,7 +23,7 @@
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
</repository>
</repositories>
<distributionManagement>
<snapshotRepository>
<id>talend_nexus_deployment</id>
@@ -46,36 +46,32 @@
</releases>
</repository>
</distributionManagement>
<dependencies>
<dependency>
<groupId>org.talend.daikon</groupId>
<artifactId>daikon-audit</artifactId>
<version>0.31.8</version>
</dependency>
<dependencies>
<dependency>
<groupId>org.talend.daikon</groupId>
<artifactId>audit-common</artifactId>
<version>1.8.0</version>
<groupId>org.talend.daikon</groupId>
<artifactId>audit-logback</artifactId>
<version>0.31.8</version>
</dependency>
<dependency>
<groupId>org.talend.daikon</groupId>
<artifactId>audit-log4j1</artifactId>
<version>1.8.0</version>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.3.0-alpha4</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.12.1</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.12.1</version>
<optional>true</optional>
</dependency>
</dependencies>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@@ -18,30 +18,13 @@ public class JobEventAuditLoggerFactory {
final AuditConfigurationMap config = AuditConfiguration.loadFromProperties(properties);
AbstractBackend logger = null;
//load log4j2 implement firstly
String loggerClass = "org.talend.logging.audit.log4j2.Log4j2Backend";
String loggerClass = "org.talend.logging.audit.logback.LogbackBackend";
try {
final Class<?> clz = Class.forName(loggerClass);
logger = (AbstractBackend) clz.getConstructor(AuditConfigurationMap.class).newInstance(config);
} catch (ReflectiveOperationException e) {
// do nothing
}
//load log4j1 implement if not found log4j2
if (logger == null) {
loggerClass = "org.talend.logging.audit.log4j1.Log4j1Backend";
try {
final Class<?> clz = Class.forName(loggerClass);
logger = (AbstractBackend) clz.getConstructor(AuditConfigurationMap.class).newInstance(config);
} catch (ReflectiveOperationException e) {
// do nothing
}
}
if(logger == null) {
throw new RuntimeException("Unable to load backend : " + loggerClass);
}
final Class<?> clz = Class.forName(loggerClass);
logger = (AbstractBackend) clz.getConstructor(AuditConfigurationMap.class).newInstance(config);
} catch (ReflectiveOperationException e) {
throw new RuntimeException("Unable to load backend " + loggerClass, e);
}
final DefaultAuditLoggerBase loggerBase = new DefaultAuditLoggerBase(logger, config);
@@ -55,4 +38,4 @@ public class JobEventAuditLoggerFactory {
return (T) Proxy.newProxyInstance(AuditLoggerFactory.class.getClassLoader(), new Class<?>[] { clz },
new ProxyEventAuditLogger(auditLoggerBase));
}
}
}

View File

@@ -8,8 +8,7 @@ public class JobAuditLoggerTest {
public static void main(String[] args) {
Properties props = new Properties();
String root_logger_name = "audit";
props.setProperty("root.logger", root_logger_name);
props.setProperty("root.logger", "audit");
props.setProperty("encoding", "UTF-8");
props.setProperty("application.name", "Talend Studio");
props.setProperty("service.name", "Talend Studio Job");
@@ -20,10 +19,6 @@ public class JobAuditLoggerTest {
props.setProperty("appender.file.maxsize", "52428800");
props.setProperty("appender.file.maxbackup", "20");
props.setProperty("host", "false");
org.apache.logging.log4j.core.config.Configurator.setLevel(root_logger_name, org.apache.logging.log4j.Level.DEBUG);
//org.apache.log4j.Logger.getLogger("audit").setLevel(org.apache.log4j.Level.DEBUG);
final JobAuditLogger logger = JobEventAuditLoggerFactory.createJobAuditLogger(props);
Context context = JobContextBuilder.create().jobName("fetch_from_s3_every_day").jobId("jobid_123")
.jobVersion("0.1").connectorType("tXMLMAP").connectorId("tXMLMap_1")
@@ -36,4 +31,4 @@ public class JobAuditLoggerTest {
logger.flowOutput(context);
logger.jobstop(context);
}
}
}

View File

@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.talend.libraries</groupId>
<artifactId>talendExcel-1.6-20210127</artifactId>
<artifactId>talendExcel-1.4-20190531</artifactId>
<version>6.0.0</version>
<packaging>jar</packaging>

View File

@@ -69,10 +69,6 @@ public class ExcelTool {
private int rowAccessWindowSize = SXSSFWorkbook.DEFAULT_WINDOW_SIZE;// used in auto flush
private boolean isTrackAllColumns = false;
private boolean isTruncateExceedingCharacters = false;
private static final int CELL_CHARACTERS_LIMIT = 32767;
public ExcelTool() {
cellStylesMapping = new HashMap<>();
@@ -278,10 +274,7 @@ public class ExcelTool {
public void addCellValue(String stringValue) {
addCell();
String value = isTruncateExceedingCharacters && stringValue != null && stringValue.length() > CELL_CHARACTERS_LIMIT
? stringValue.substring(0, CELL_CHARACTERS_LIMIT)
: stringValue;
curCell.setCellValue(value);
curCell.setCellValue(stringValue);
curCell.setCellStyle(getNormalCellStyle());
}
@@ -356,8 +349,4 @@ public class ExcelTool {
((SXSSFSheet) sheet).flushRows();
}
}
public void setTruncateExceedingCharacters(boolean isTruncateExceedingCharacters) {
this.isTruncateExceedingCharacters = isTruncateExceedingCharacters;
}
}

View File

@@ -1,10 +1,10 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.talend.components.lib</groupId>
<groupId>org.talend.libraries</groupId>
<artifactId>talendsap</artifactId>
<name>talendsap</name>
<version>1.0.2</version>
<version>1.0.1</version>
<properties>
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>

View File

@@ -90,7 +90,6 @@ public class DocumentExtractor {
List<Element> tablesAndChangingElements = new ArrayList<Element>(3);
tablesAndChangingElements.add(functionElement.element("TABLES"));
tablesAndChangingElements.add(functionElement.element("CHANGING"));
tablesAndChangingElements.add(functionElement.element("OUTPUT"));
for(Element tablesOrChangingElement : tablesAndChangingElements) {
if (tablesOrChangingElement == null) {
@@ -122,10 +121,6 @@ public class DocumentExtractor {
}
result.add(row);
}
if(!result.isEmpty()) {
return result;
}
}
return result;

View File

@@ -49,11 +49,11 @@ public class DocumentHelper {
}
}
public void addSingleParameter(String name, String value, SAPParameterType parameter_type) {
public void addSingleParameter(String name, String value, boolean isChanging) {
if(value == null) {
value = "";
}
if (parameter_type == SAPParameterType.CHANGING) {
if (isChanging) {
correctChanging();
changing.addElement(name).setText(value);
} else {
@@ -62,8 +62,8 @@ public class DocumentHelper {
}
}
public void addStructParameter(String name, SAPParameterType parameter_type) {
if (parameter_type == SAPParameterType.CHANGING) {
public void addStructParameter(String name, boolean isChanging) {
if (isChanging) {
correctChanging();
currentStruct = changing.addElement(name);
} else {
@@ -79,16 +79,13 @@ public class DocumentHelper {
currentStruct.addElement(name).setText(value);
}
public void addTableParameter(String name, SAPParameterType parameter_type) {
if (parameter_type == SAPParameterType.CHANGING) {
public void addTableParameter(String name, boolean isChanging) {
if (isChanging) {
correctChanging();
currentTable = changing.addElement(name);
} else if(parameter_type == SAPParameterType.TABLES) {
} else {
correctTables();
currentTable = tables.addElement(name);
} else {
correctInput();
currentTable = input.addElement(name);
}
}
@@ -111,18 +108,18 @@ public class DocumentHelper {
DocumentHelper helper = new DocumentHelper();
helper.setFunctionName("READ_TABLE_FUNCTION");
helper.addSingleParameter("ID", "1", SAPParameterType.CHANGING);
helper.addSingleParameter("NAME", "gaoyan", SAPParameterType.IMPORT);
helper.addSingleParameter("ID", "1", true);
helper.addSingleParameter("NAME", "gaoyan", false);
helper.addStructParameter("INFO", SAPParameterType.CHANGING);
helper.addStructParameter("INFO", true);
helper.addStructChildParameter("ID", "2");
helper.addStructChildParameter("NAME", "wangwei");
helper.addStructParameter("INFO1", SAPParameterType.IMPORT);
helper.addStructParameter("INFO1", false);
helper.addStructChildParameter("ID1", "4");
helper.addStructChildParameter("NAME1", "momo");
helper.addTableParameter("TABLE1", SAPParameterType.TABLES);
helper.addTableParameter("TABLE1", false);
for (int i = 0; i < 200000; i++) {
helper.addTableRow();
helper.addTableRowChildParameter("c1", i + "");
@@ -135,7 +132,7 @@ public class DocumentHelper {
helper.addTableRowChildParameter("c8", "wangwei" + i);
}
helper.addTableParameter("TABLE2", SAPParameterType.TABLES);
helper.addTableParameter("TABLE2", false);
for (int i = 0; i < 2; i++) {
helper.addTableRow();
helper.addTableRowChildParameter("ID4", i + "");

View File

@@ -1,8 +0,0 @@
package org.talend.sap;
public enum SAPParameterType {
IMPORT,
CHANGING,
TABLES,
EXPORT
}

View File

@@ -46,7 +46,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.19</version>
<version>1.10</version>
</dependency>
<dependency>
@@ -55,4 +55,4 @@
<version>6.0.0</version>
</dependency>
</dependencies>
</project>
</project>

View File

@@ -5,9 +5,9 @@
>
<modelVersion>4.0.0</modelVersion>
<groupId>org.talend.components.lib</groupId>
<groupId>org.talend.libraries</groupId>
<artifactId>thashfile</artifactId>
<version>3.1-20190910</version>
<version>3.0-20170711</version>
<packaging>jar</packaging>
<name>thashfile</name>
@@ -45,8 +45,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>

View File

@@ -9,44 +9,45 @@ import java.util.concurrent.ConcurrentHashMap;
import org.talend.designer.components.hashfile.memory.AdvancedMemoryHashFile;
public class MapHashFile {
// use this map instead of globalMap
//use this map instead of globalMap
private Map<String, AdvancedMemoryHashFile> resourceMap = new ConcurrentHashMap<>();
// keep the present key of AdvancedMemoryHashFile as key and the previous key as
// value
//keep the present key of AdvancedMemoryHashFile as key and the previous key as value
private Map<String, String> keyMap = new ConcurrentHashMap<>();
// singleton
//singleton
private static final MapHashFile mhf = new MapHashFile();
public static TalendMultiThreadLockMap resourceLockMap = new TalendMultiThreadLockMap();
public static TalendMultiThreadLockMap resourceLockMap = new TalendMultiThreadLockMap();
public static class TalendMultiThreadLockMap {
public static class TalendMultiThreadLockMap {
private Map<Object, Object> tMultiTheadLockMap = new ConcurrentHashMap<>();
public Object get(Object key) {
return tMultiTheadLockMap.computeIfAbsent(key, k -> new Object());
}
public void remove(Object key) {
tMultiTheadLockMap.remove(key);
}
}
private Map<Object, Object> tMultiTheadLockMap = new HashMap<Object, Object>();
public synchronized Object get(Object key) {
if (tMultiTheadLockMap.get(key) == null) {
tMultiTheadLockMap.put(key, new Object());
}
return tMultiTheadLockMap.get(key);
}
public synchronized void remove(Object key){
tMultiTheadLockMap.remove(key);
}
}
private MapHashFile() {
}
public static MapHashFile getMapHashFile() {
return mhf;
}
// get the linked AdvancedMemoryHashFile
//get the linked AdvancedMemoryHashFile
public AdvancedMemoryHashFile getAdvancedMemoryHashFile(String key) {
AdvancedMemoryHashFile amhf = resourceMap.get(key);
String prekey = keyMap.get(key);
// if present AdvancedMemoryHashFile is null get the AdvancedMemoryHashFile
// before present.
//if present AdvancedMemoryHashFile is null get the AdvancedMemoryHashFile before present.
int size = keyMap.size();
while (amhf == null && (size--) > 0) {
while(amhf==null && (size--)>0){
amhf = resourceMap.get(prekey);
prekey = keyMap.get(prekey);
}
@@ -60,18 +61,16 @@ public class MapHashFile {
public Map<String, String> getKeyMap() {
return keyMap;
}
public void clearCache(String key) {
public void clearCache(String key){
clearChildCache(getRootCache(key));
}
public void clearChildCache(String root) {
public void clearChildCache(String root){
Set<String> set = keyMap.keySet();
synchronized (keyMap) {
synchronized(keyMap) {
Iterator<String> it = set.iterator();
while (it.hasNext()) {
while(it.hasNext()){
String key = it.next();
if (root.equals(keyMap.get(key))) {
if(root.equals(keyMap.get(key))){
this.resourceMap.remove(key);
this.keyMap.remove(key);
clearChildCache(key);
@@ -80,11 +79,11 @@ public class MapHashFile {
}
this.resourceMap.remove(root);
}
public String getRootCache(String cache) {
public String getRootCache(String cache){
String root;
while ((root = keyMap.get(cache)) != null) {
cache = root;
while((root = keyMap.get(cache))!=null){
cache=root;
}
return cache;
}

View File

@@ -54,11 +54,11 @@ for(IConnection conn : outgoingConns) {
<%
}
}
log4jCodeGenerateUtil.query(node);
%>
query_<%=cid %> = <%=dbquery%>;
whetherReject_<%=cid%> = false;
<%
log4jCodeGenerateUtil.query(node, "query_" + cid);
List<IMetadataTable> metadatas = node.getMetadataList();
if ((metadatas!=null)&&(metadatas.size()>0)) {
IMetadataTable metadata = metadatas.get(0);
@@ -131,7 +131,7 @@ try {
<%
}
}
log4jCodeGenerateUtil.logInfo(node,"info",cid+" - Execute the query: '\" + query_" + cid + " + \"' has finished.");
log4jCodeGenerateUtil.logInfo(node,"info",cid+" - Execute the query: '\" + "+dbquery +" + \"' has finished.");
%>
<% //feature 0010425
if(usePrepareStatement){

View File

@@ -24,9 +24,6 @@
String user = ElementParameterParser.getValue(node, "__USER__");
String node_type = ElementParameterParser.getValue(node, "__NODE_TYPE__");
String node_count = ElementParameterParser.getValue(node, "__NODE_COUNT__");
if (node_count.isEmpty()) {
node_count = "1";
}
boolean isLog4jEnabled = "true".equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
@@ -91,46 +88,52 @@
.withMasterUsername(<%=user%>)
.withMasterUserPassword(decryptedPwd_<%=cid%>)
.withNodeType(<%=node_type%>);
.withNodeType(<%=node_type%>)
if (<%=node_count%> > 1) {
request_<%=cid%> = request_<%=cid%>.withNumberOfNodes(<%=node_count%>);
} else {
request_<%=cid%> = request_<%=cid%>.withClusterType("single-node");
}
//advanced settings
<%if(parameterUtil.isValid(parameter_group_name)) {%>
request_<%=cid%> = request_<%=cid%>.withClusterParameterGroupName(<%=parameter_group_name%>);
<%}%>
<%if(parameterUtil.isValid(subnet_group_name)) {%>
request_<%=cid%> = request_<%=cid%>.withClusterSubnetGroupName(<%=subnet_group_name%>);
<%}%>
<%
if(publicly_accessible) {
%>
request_<%=cid%> = request_<%=cid%>.withPubliclyAccessible(true);
<%if(set_public_ip_address) {%>
request_<%=cid%> = request_<%=cid%>.withElasticIp(<%=elastic_ip%>);
<%
if(new Integer(node_count) > 1){
%>
.withNumberOfNodes(<%=node_count%>)
<%
} else {
%>
.withClusterType("single-node")
<%
}
%>
//advanced settings
<%if(parameterUtil.isValid(parameter_group_name)) {%>
.withClusterParameterGroupName(<%=parameter_group_name%>)
<%}%>
<%if(parameterUtil.isValid(subnet_group_name)) {%>
.withClusterSubnetGroupName(<%=subnet_group_name%>)
<%}%>
<%
} else {
%>
request_<%=cid%> = request_<%=cid%>.withPubliclyAccessible(false);
<%
}
%>
<%if(parameterUtil.isValid(availability_zone)) {%>
request_<%=cid%> = request_<%=cid%>.withAvailabilityZone(<%=availability_zone%>);
<%}%>
<%if(parameterUtil.isValid(vpc_security_groupids)) {%>
request_<%=cid%> = request_<%=cid%>.withVpcSecurityGroupIds(<%=vpc_security_groupids%>.split(","));
<%}%>
<%
if(publicly_accessible) {
%>
.withPubliclyAccessible(true)
<%if(set_public_ip_address) {%>
.withElasticIp(<%=elastic_ip%>)
<%}%>
<%
} else {
%>
.withPubliclyAccessible(false)
<%
}
%>
<%if(parameterUtil.isValid(availability_zone)) {%>
.withAvailabilityZone(<%=availability_zone%>)
<%}%>
<%if(parameterUtil.isValid(vpc_security_groupids)) {%>
.withVpcSecurityGroupIds(<%=vpc_security_groupids%>.split(","))
<%}%>
;
com.amazonaws.services.redshift.model.Cluster result_<%=cid%> = client_<%=cid%>.createCluster(request_<%=cid%>);
<%if(isLog4jEnabled) {%>

View File

@@ -149,7 +149,7 @@
<IMPORTS>
<IMPORT
NAME="Talen File Enhanced"
MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar"
MODULE="talend_file_enhanced_20070724.jar" MVN="mvn:org.talend.libraries/talend_file_enhanced_20070724/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced_20070724.jar"
REQUIRED="true"
/>
</IMPORTS>

View File

@@ -32,8 +32,6 @@
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
boolean useCustomNullMarker = ElementParameterParser.getBooleanValue(node, "__USE_CUSTOM_NULL_MARKER__");
String nullMarker = useCustomNullMarker ? ElementParameterParser.getValue(node, "__CUSTOM_NULL_MARKER__") : "\"\\\\N\"";
String passwordFieldName = "";
if (authMode.equals("OAUTH")) {
@@ -362,7 +360,7 @@
queryLoad_<%=cid%>.setDestinationTable(destinationTable_<%=cid%>);
queryLoad_<%=cid%>.setSourceUris(java.util.Arrays.asList(<%=ElementParameterParser.getValue(node, "__GS_FILE__")%>));
queryLoad_<%=cid%>.setSkipLeadingRows(<%=ElementParameterParser.getValue(node, "__GS_FILE_HEADER__")%>);
queryLoad_<%=cid%>.setNullMarker(<%= nullMarker %>);
queryLoad_<%=cid%>.setNullMarker("\\N");
config_<%=cid%>.setLoad(queryLoad_<%=cid%>);
job_<%=cid%>.setConfiguration(config_<%=cid%>);
@@ -538,6 +536,7 @@
com.google.cloud.bigquery.Schema schema_<%=cid%> = com.google.cloud.bigquery.Schema.of(fields_<%=cid%>);
com.google.cloud.bigquery.TableInfo tableInfo_<%=cid%> = com.google.cloud.bigquery.TableInfo.newBuilder(tableId_<%=cid%>, com.google.cloud.bigquery.StandardTableDefinition.of(schema_<%=cid%>)).build();
table_<%=cid%> = bigquery_<%=cid%>.create(tableInfo_<%=cid%>);
loadJobBuilder_<%=cid%>.setSchema(schema_<%=cid%>);
}
@@ -574,16 +573,15 @@
loadJobBuilder_<%=cid%>.setDestinationTable(tableId_<%=cid%>);
com.google.cloud.bigquery.CsvOptions.Builder csvOptions_<%=cid%> = com.google.cloud.bigquery.CsvOptions.newBuilder();
csvOptions_<%=cid%>.setAllowQuotedNewLines(true);
csvOptions_<%=cid%>.setSkipLeadingRows(<%=ElementParameterParser.getValue(node, "__GS_FILE_HEADER__")%>);
<%if("true".equals(ElementParameterParser.getValue(node, "__SET_FIELD_DELIMITER__"))) {
%>
csvOptions_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>);
loadJobBuilder_<%=cid%>.setFormatOptions(csvOptions_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>).build());
<%
}
%>
loadJobBuilder_<%=cid%>.setFormatOptions(csvOptions_<%=cid%>.build());
loadJobBuilder_<%=cid%>.setNullMarker(<%= nullMarker %>);
loadJobBuilder_<%=cid%>.setNullMarker("\\N");
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.of(loadJobBuilder_<%=cid%>.build()));
job_<%=cid%> = job_<%=cid%>.waitFor(com.google.cloud.RetryOption.initialRetryDelay(org.threeten.bp.Duration.ofSeconds(1)));
if (job_<%=cid%> != null && job_<%=cid%>.getStatus().getError() == null) {

View File

@@ -173,16 +173,10 @@
SHOW_IF="(SET_FIELD_DELIMITER == 'true')">
<DEFAULT>","</DEFAULT>
</PARAMETER>
<PARAMETER NAME="USE_CUSTOM_NULL_MARKER" FIELD="CHECK" NUM_ROW="25">
<DEFAULT>false</DEFAULT>
</PARAMETER>
<PARAMETER NAME="CUSTOM_NULL_MARKER" FIELD="TEXT" NUM_ROW="25" SHOW_IF="USE_CUSTOM_NULL_MARKER=='true'">
<DEFAULT>""</DEFAULT>
</PARAMETER>
<PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="30" SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'">
<PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="25" SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'">
<DEFAULT>false</DEFAULT>
</PARAMETER>
<PARAMETER NAME="ENCODING" FIELD="ENCODING_TYPE" NUM_ROW="35"
<PARAMETER NAME="ENCODING" FIELD="ENCODING_TYPE" NUM_ROW="30"
REQUIRED="true" REPOSITORY_VALUE="ENCODING">
<DEFAULT>"ISO-8859-15"</DEFAULT>
</PARAMETER>

View File

@@ -40,8 +40,6 @@ ENCODING.NAME=Encoding
TOKEN_NAME.NAME=token properties File Name
SET_FIELD_DELIMITER.NAME=Set the field delimiter
FIELD_DELIMITER.NAME=
USE_CUSTOM_NULL_MARKER.NAME=Use custom null marker
CUSTOM_NULL_MARKER.NAME=
DROP.NAME=Drop table if exists
DIE_ON_ERROR.NAME=Die on error

View File

@@ -21,7 +21,7 @@ class BigQueryUtil_<%=cid%> {
private String genTempName(String prefix){
return "temp_" + prefix + java.util.UUID.randomUUID().toString().replaceAll("-", "") + "<%=cid%>".toLowerCase().replaceAll("[^a-z0-9]", "0").replaceAll("^[^a-z]", "a") + Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
}
public void cleanup() throws Exception{
if(useLargeResult){
bigqueryclient.tables().delete(projectId, tempDataset, tempTable).execute();
@@ -38,11 +38,11 @@ class BigQueryUtil_<%=cid%> {
job.setConfiguration(config);
List<com.google.api.services.bigquery.model.TableReference> referencedTables = bigqueryclient.jobs().insert(projectId, job).execute().getStatistics().getQuery().getReferencedTables();
if(referencedTables != null && !referencedTables.isEmpty()) {
location = bigqueryclient.tables().get(referencedTables.get(0).getProjectId(), referencedTables.get(0).getDatasetId(), referencedTables.get(0).getTableId()).execute().getLocation();
location = bigqueryclient.tables().get(projectId, referencedTables.get(0).getDatasetId(), referencedTables.get(0).getTableId()).execute().getLocation();
}
return location;
}
private void createDataset(String location) throws Exception {
com.google.api.services.bigquery.model.Dataset dataset = new com.google.api.services.bigquery.model.Dataset().setDatasetReference(new com.google.api.services.bigquery.model.DatasetReference().setProjectId(projectId).setDatasetId(tempDataset));
if(location != null) {
@@ -53,11 +53,11 @@ class BigQueryUtil_<%=cid%> {
dataset.setDescription(description);
bigqueryclient.datasets().insert(projectId, dataset).execute();
}
public com.google.api.services.bigquery.model.Job executeQuery(String query, boolean useLargeResult) throws Exception{
return executeQuery(query, useLargeResult, true);
}
public com.google.api.services.bigquery.model.Job executeQuery(String query, boolean useLargeResult, boolean useLegacySql) throws Exception{
com.google.api.services.bigquery.model.JobConfigurationQuery queryConfig = new com.google.api.services.bigquery.model.JobConfigurationQuery();
queryConfig.setQuery(query);
@@ -74,13 +74,13 @@ class BigQueryUtil_<%=cid%> {
.setDatasetId(tempDataset)
.setTableId(tempTable));
}
com.google.api.services.bigquery.model.JobConfiguration config = new com.google.api.services.bigquery.model.JobConfiguration();
config.setQuery(queryConfig);
com.google.api.services.bigquery.model.Job job = new com.google.api.services.bigquery.model.Job();
job.setConfiguration(config);
com.google.api.services.bigquery.model.Job insert = null;
com.google.api.services.bigquery.model.JobReference jobId = null;
try {
@@ -122,7 +122,7 @@ class BigQueryUtil_<%=cid%> {
}
throw e;
}
<%
if(isLog4jEnabled){
%>
@@ -146,7 +146,7 @@ class BigQueryUtil_<%=cid%> {
<%
}
%>
return executeQuery(query, true, useLegacySql);
return executeQuery(query, true);
}
<%}%>
// Do not throw exception to avoid behavior changed(because it may throw "duplicate" exception which do not throw before);
@@ -164,13 +164,13 @@ class BigQueryUtil_<%=cid%> {
}// else job successful
break;
}
// Pause execution for one second before polling job status again, to
// Pause execution for one second before polling job status again, to
// reduce unnecessary calls to the BigQUery API and lower overall
// application bandwidth.
Thread.sleep(1000);
}
return insert;
}
}
}

View File

@@ -221,9 +221,10 @@
}
String query_<%=cid%> = <%=query%>;
<% if(isLog4jEnabled) { %>
log.info("<%=cid%> - query " + <%=query%>);
<% } %>
String dataset = query_<%=cid%>.substring(query_<%=cid%>.indexOf("from") + 4, query_<%=cid%>.indexOf(".")).trim();
String temp_table = "temp_" + dataset + java.util.UUID.randomUUID().toString().replaceAll("-", "")
+ "<%=cid%>".toLowerCase().replaceAll("[^a-z0-9]", "0").replaceAll("^[^a-z]", "a")
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
com.google.cloud.bigquery.BigQuery bigquery_<%=cid%> = com.google.cloud.bigquery.BigQueryOptions.newBuilder()
.setCredentials(credentials_<%=cid%>)
@@ -232,55 +233,25 @@
.getService();
com.google.cloud.bigquery.QueryJobConfiguration.Builder queryConfiguration_<%=cid%> = com.google.cloud.bigquery.QueryJobConfiguration.newBuilder(<%=query%>).setUseLegacySql(<%=useLegacySql%>);
<%
if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) {
%>
queryConfiguration_<%=cid%>.setDestinationTable(com.google.cloud.bigquery.TableId.of(dataset, temp_table)).setAllowLargeResults(true);
<%
}
%>
com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId.of(java.util.UUID.randomUUID().toString());
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.newBuilder(queryConfiguration_<%=cid%>.build()).setJobId(jobId_<%=cid%>).build());
<%if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) {%>
com.google.cloud.bigquery.QueryJobConfiguration jobConfDryRun_<%=cid%> = queryConfiguration_<%=cid%> .setDryRun(true).build();
com.google.cloud.bigquery.Job jobDryRun_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.of(jobConfDryRun_<%=cid%>));
String queryLocation_<%=cid%> =jobDryRun_<%=cid%>.getJobId().getLocation();
String location_<%=cid%> = queryLocation_<%=cid%> == null ? "US" : queryLocation_<%=cid%>;
String tempDataset_<%=cid%> = java.util.UUID.randomUUID().toString().replaceAll("-", "")
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt())
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
String tempTable_<%=cid%> = java.util.UUID.randomUUID().toString().replaceAll("-", "")
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt())
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
<% if(isLog4jEnabled){ %>
log.info("<%=cid%> - query location :" + queryLocation_<%=cid%>);
log.info("<%=cid%> - temporary dataset location :" + location_<%=cid%>);
log.info("<%=cid%> - temporary Dataset name : " + tempDataset_<%=cid%>);
log.info("<%=cid%> - temporary table name : " + tempTable_<%=cid%>);
<% } %>
com.google.cloud.bigquery.DatasetInfo datasetInfo_<%=cid%> = com.google.cloud.bigquery.DatasetInfo.newBuilder(tempDataset_<%=cid%>).setLocation(location_<%=cid%>).build();
com.google.cloud.bigquery.Dataset dataset_<%=cid%> = bigquery_<%=cid%> .create(datasetInfo_<%=cid%>);
queryConfiguration_<%=cid%>
.setDryRun(false)
.setAllowLargeResults(true)
.setDestinationTable(com.google.cloud.bigquery.TableId.of(tempDataset_<%=cid%>, tempTable_<%=cid%>));
com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId
.newBuilder().setProject(<%=projectId%>)
.setJob(java.util.UUID.randomUUID().toString())
.setLocation(location_<%=cid%>)
.build();
<% if(isLog4jEnabled){ %>
log.info("<%=cid%> - job location : " + jobId_<%=cid%>.getLocation());
<% } %>
<% } else { %>
com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId.of(<%=projectId%>,java.util.UUID.randomUUID().toString());
<% } %>
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.newBuilder(queryConfiguration_<%=cid%>.build()).setJobId(jobId_<%=cid%>).build());
<% if(isLog4jEnabled) { %>
<%
if(isLog4jEnabled){
%>
log.info("<%=cid%> - Sending job " + jobId_<%=cid%> + " with query: " + <%=query%>);
<% } %>
<%
}
%>
job_<%=cid%> = job_<%=cid%>.waitFor();
if (job_<%=cid%> == null) {
@@ -289,9 +260,13 @@
throw new RuntimeException(job_<%=cid%>.getStatus().getError().toString());
}
<% if(isLog4jEnabled) { %>
<%
if(isLog4jEnabled){
%>
log.info("<%=cid%> - Job " + jobId_<%=cid%> + " finished successfully.");
<% } %>
<%
}
%>
com.google.cloud.bigquery.TableResult result_<%=cid%> = job_<%=cid%>.getQueryResults();
long nb_line_<%=cid%> = 0;
<%

View File

@@ -10,10 +10,7 @@ imports="
CodeGeneratorArgument codeGenArgument = (CodeGeneratorArgument) argument;
INode node = (INode)codeGenArgument.getArgument();
String cid = node.getUniqueName();
String authMode = ElementParameterParser.getValue(node,"__AUTH_MODE__");
String projectId = ElementParameterParser.getValue(node,"__PROJECT_ID__");
String resultSizeType = ElementParameterParser.getValue(node,"__RESULT_SIZE__");
String authMode = ElementParameterParser.getValue(node,"__AUTH_MODE__");
if (authMode.equals("OAUTH")) {
%>
}
@@ -27,10 +24,7 @@ imports="
} else if (authMode.equals("SERVICEACCOUNT")) {
%>
}
<% if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) { %>
com.google.cloud.bigquery.DatasetId datasetId_<%=cid%> = com.google.cloud.bigquery.DatasetId.of(<%=projectId%>, tempDataset_<%=cid%>);
bigquery_<%=cid%>.delete(datasetId_<%=cid%>, com.google.cloud.bigquery.BigQuery.DatasetDeleteOption.deleteContents());
<% }%>
bigquery_<%=cid%>.delete(com.google.cloud.bigquery.TableId.of(dataset, temp_table));
<%
} else {
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);

View File

@@ -55,12 +55,7 @@
<PARAMETER NAME="TEXT_ENCLOSURE" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" MAX_LENGTH="1" REPOSITORY_VALUE="TEXT_ENCLOSURE" SHOW="false">
<DEFAULT>"\""</DEFAULT>
</PARAMETER>
<PARAMETER NAME="USE_CUSTOM_NULL_MARKER" FIELD="CHECK" NUM_ROW="35">
<DEFAULT>false</DEFAULT>
</PARAMETER>
<PARAMETER NAME="CUSTOM_NULL_MARKER" FIELD="TEXT" NUM_ROW="35" SHOW_IF="USE_CUSTOM_NULL_MARKER=='true'">
<DEFAULT>""</DEFAULT>
</PARAMETER>
<PARAMETER NAME="CREATE" FIELD="CHECK" REQUIRED="true" NUM_ROW="40">
<DEFAULT>true</DEFAULT>
</PARAMETER>

View File

@@ -25,8 +25,7 @@
boolean checkDiskSpace = ("true").equals(ElementParameterParser.getValue(node,"__CHECK_DISK_SPACE__"));
boolean flushOnRow = ("true").equals(ElementParameterParser.getValue(node, "__FLUSHONROW__"));
String flushMod = ElementParameterParser.getValue(node, "__FLUSHONROW_NUM__");
boolean useCustomNullMarker = ElementParameterParser.getBooleanValue(node, "__USE_CUSTOM_NULL_MARKER__");
String nullMarker = useCustomNullMarker ? ElementParameterParser.getValue(node, "__CUSTOM_NULL_MARKER__") : "\"\\\\N\"";
String encoding = ElementParameterParser.getValue(node,"__ENCODING__");
List< ? extends IConnection> conns = node.getIncomingConnections();
@@ -36,8 +35,8 @@
if (conn.getLineStyle().hasConnectionCategory(IConnectionCategory.DATA)) {
List<IMetadataColumn> columns = metadata.getListColumns();
int sizeColumns = columns.size();
%>
String[] row_<%=cid%>=new String[]{<%for(int j=0;j<sizeColumns;j++){%> <%= nullMarker %>,<%}%>}; //empty value must be defined by null marker('\N' is default in bulk file)
%>
String[] row_<%=cid%>=new String[]{<%for(int j=0;j<sizeColumns;j++){%>"\\N",<%}%>}; //empty value must be NULL('\N' in bulk file)
<%
for (int i = 0; i < sizeColumns; i++) {
IMetadataColumn column = columns.get(i);
@@ -65,10 +64,24 @@
%>
row_<%=cid%>[<%=i%>] = <%=conn.getName() %>.<%=column.getLabel() %>;
<%
}else if(javaType == JavaTypesManager.DATE){
%>
row_<%=cid%>[<%=i%>] = FormatterUtils.format_Date(<%=conn.getName() %>.<%=column.getLabel() %>, <%=(pattern != null && !("\"\"").equals(pattern)) ? pattern : "\"yyyy-MM-dd\""%>);
<%
}else if(javaType == JavaTypesManager.DATE && pattern == null){
%>
row_<%=cid%>[<%=i%>] = FormatterUtils.format_Date(<%=conn.getName() %>.<%=column.getLabel() %>, "yyyy-MM-dd");
<%
}else if(javaType == JavaTypesManager.DATE && pattern != null){
if(pattern.length() > 12){
%>
row_<%=cid%>[<%=i%>] = FormatterUtils.format_Date(<%=conn.getName() %>.<%=column.getLabel() %>, "yyyy-MM-dd HH:mm:ss");
<%
}else if(pattern.length() == 12 || "\"\"".equals(pattern)) {
%>
row_<%=cid%>[<%=i%>] = FormatterUtils.format_Date(<%=conn.getName() %>.<%=column.getLabel() %>, "yyyy-MM-dd");
<%
}else {
%>
row_<%=cid%>[<%=i%>] = FormatterUtils.format_Date(<%=conn.getName() %>.<%=column.getLabel() %>, <%=pattern%>);
<%
}
}else if(javaType == JavaTypesManager.BYTE_ARRAY){
%>
row_<%=cid%>[<%=i%>] = java.nio.charset.Charset.forName(<%=encoding %>).decode(java.nio.ByteBuffer.wrap(<%=conn.getName() %>.<%=column.getLabel() %>)).toString();
@@ -87,7 +100,7 @@
}
%>
} else {
row_<%=cid%>[<%=i%>] = <%= nullMarker %>;
row_<%=cid%>[<%=i%>] = "\\N";
}
<%
}

View File

@@ -24,5 +24,4 @@ RECORDS_CONTAIN_NULL_VALUE.NAME=Records contain NULL value
PROPERTY.NAME=Property Type
CHECK_DISK_SPACE.NAME=Check disk space
USE_CUSTOM_NULL_MARKER.NAME=Use custom null marker
CUSTOM_NULL_MARKER.NAME=

View File

@@ -17,40 +17,40 @@ imports="
CodeGeneratorArgument codeGenArgument = (CodeGeneratorArgument) argument;
INode node = (INode)codeGenArgument.getArgument();
String cid = node.getUniqueName();
String useExistingConn = ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__");
String accessToken = ElementParameterParser.getValue(node, "__ACCESS_TOKEN__");
String refreshToken = ElementParameterParser.getValue(node, "__REFRESH_TOKEN__");
String clientId = ElementParameterParser.getValue(node, "__CLIENT_ID__");
String clientSecret = ElementParameterParser.getValue(node, "__CLIENT_SECRET__");
String connection = ElementParameterParser.getValue(node,"__CONNECTION__");
String connectionKey = "\"conn_" + connection+"\"";
String path = ElementParameterParser.getValue(node, "__PATH__");
String filelistType = ElementParameterParser.getValue(node, "__LIST_MODE__");
boolean includeSubDirectories = "true".equals(ElementParameterParser.getValue(node, "__INCLUDSUBDIR__"));
final boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
log4jFileUtil.componentStartInfo(node);
List<IMetadataTable> metadatas = node.getMetadataList();
IMetadataTable metadata = null;
if ((metadatas!=null)&&(metadatas.size()>0)) {
metadata = metadatas.get(0);
metadata = metadatas.get(0);
}
List< ? extends IConnection> outputConnections = node.getOutgoingSortedConnections();
String dataOutputConnection = null;
String dataOutputConnection = null;
for(IConnection conn : outputConnections) {
if(conn.getLineStyle().hasConnectionCategory(IConnectionCategory.DATA)) {
dataOutputConnection = conn.getName();
break;
} // if(conn) end
} // for(conns) end
%>
<%
if(!"true".equals(useExistingConn)){
@@ -105,81 +105,115 @@ if(!"true".equals(useExistingConn)){
<%=cid%>_client.authenticate(new com.box.boxjavalibv2.dao.BoxOAuthToken(<%=cid%>_map));
}
<%
} else {
} else {
%>
com.box.boxjavalibv2.BoxClient <%=cid%>_client = (com.box.boxjavalibv2.BoxClient)globalMap.get(<%=connectionKey%>);
<%
}
%>
String <%=cid%>_path = normalizePath(<%=path%>);
com.box.boxjavalibv2.dao.BoxFolder <%=cid%>_rootFolder = <%=cid%>_client.getFoldersManager().getFolder("0", null);
com.box.boxjavalibv2.dao.BoxItem <%=cid%>_boxItem;
int <%=cid%>_slashPosition = <%=cid%>_path.indexOf("/");
String <%=cid%>_rootFolderName = <%=cid%>_slashPosition == -1 ? <%=cid%>_path : <%=cid%>_path.substring(0, <%=cid%>_slashPosition);
if (<%=cid%>_rootFolderName.equals("All Files")) {
List<String> <%=cid%>_pathItems = new java.util.ArrayList<String>(java.util.Arrays.asList(<%=cid%>_path.split("/")));
<%=cid%>_pathItems.remove(0);
try {
<%=cid%>_boxItem = getBoxItemRecursively(<%=cid%>_pathItems, <%=cid%>_rootFolder, <%=cid%>_client);
} catch (Exception e) {
throw new Exception("<%=cid%> - " + "The path to file or directory was not found in " + <%=cid%>_path, e);
}
} else {
throw new Exception("<%=cid%> - " + "The path to file or directory was not found in " + <%=cid%>_path);
boolean <%=cid%>_hasError = false;
String <%=cid%>_name = getFolderFromPath(<%=cid%>_path);
com.box.boxjavalibv2.dao.BoxFile <%=cid%>_boxFile = null;
com.box.boxjavalibv2.dao.BoxFolder <%=cid%>_boxFolder = null;
if ((<%=cid%>_name.equals("All Files") && (<%=cid%>_path.isEmpty() || <%=cid%>_path == null)))
{
<%=cid%>_boxFolder = <%=cid%>_client.getFoldersManager().getFolder("0",null);
}
java.util.List<com.box.boxjavalibv2.dao.BoxItem> <%=cid%>_items = new java.util.ArrayList<>();
if (<%=cid%>_boxItem.getType().equals("folder")) {
com.box.boxjavalibv2.dao.BoxFolder <%=cid%>_boxFolder = (com.box.boxjavalibv2.dao.BoxFolder) <%=cid%>_boxItem;
<%if(includeSubDirectories) {%>
<%if(filelistType.equalsIgnoreCase("Directories")) {%>
<%=cid%>_items = getChildrenRecursively(<%=cid%>_client, <%=cid%>_boxFolder, <%=cid%>_items, true, false);
<%} else if (filelistType.equalsIgnoreCase("Files")) {%>
<%=cid%>_items = getChildrenRecursively(<%=cid%>_client, <%=cid%>_boxFolder, <%=cid%>_items, false, true);
<%} else {%>
<%=cid%>_items = getChildrenRecursively(<%=cid%>_client, <%=cid%>_boxFolder, <%=cid%>_items, true, true);
<%}%>
<%} else {%>
for (com.box.boxjavalibv2.dao.BoxTypedObject obj : <%=cid%>_boxFolder.getItemCollection().getEntries())
else
{
com.box.restclientv2.requestsbase.BoxDefaultRequestObject <%=cid%>_requestObject = new com.box.restclientv2.requestsbase.BoxDefaultRequestObject();
if (<%=cid%>_name.equals("All Files") || <%=cid%>_path.equals("All Files"))
{
<%if(filelistType.equalsIgnoreCase("Directories")) {%>
if(obj.getType().equals("folder"))
{
com.box.boxjavalibv2.dao.BoxFolder folder = <%=cid%>_client.getFoldersManager().getFolder(obj.getId(), null);
<%=cid%>_items.add(folder);
}
<%} else if (filelistType.equalsIgnoreCase("Files")) {%>
if (obj.getType().equals("file"))
{
com.box.boxjavalibv2.dao.BoxFile file = <%=cid%>_client.getFilesManager().getFile(obj.getId(), null);
<%=cid%>_items.add(file);
}
<%} else {%>
if(obj.getType().equals("folder"))
{
com.box.boxjavalibv2.dao.BoxFolder folder = <%=cid%>_client.getFoldersManager().getFolder(obj.getId(), null);
<%=cid%>_items.add(folder);
}
else if (obj.getType().equals("file"))
{
com.box.boxjavalibv2.dao.BoxFile file = <%=cid%>_client.getFilesManager().getFile(obj.getId(), null);
<%=cid%>_items.add(file);
}
<%}%>
<%=cid%>_boxFolder = <%=cid%>_client.getFoldersManager().getFolder("0", null);
}
else
{
String parentPath = normalizePath(<%=cid%>_path.replace(<%=cid%>_name, ""));
<%=cid%>_boxFolder = <%=cid%>_client.getFoldersManager().getFolder("0", <%=cid%>_requestObject);
if ("All Files".equals(parentPath)) {
for (com.box.boxjavalibv2.dao.BoxTypedObject <%=cid%>_boxTypedObject : <%=cid%>_boxFolder.getItemCollection().getEntries()) {
if (<%=cid%>_boxTypedObject.getValue("name").equals(<%=cid%>_name)) {
if ("folder".equals(<%=cid%>_boxTypedObject.getType())) {
<%=cid%>_boxFolder = <%=cid%>_client.getFoldersManager().getFolder(<%=cid%>_boxTypedObject.getId(), <%=cid%>_requestObject);
} else {
<%=cid%>_boxFile = <%=cid%>_client.getFilesManager().getFile(<%=cid%>_boxTypedObject.getId(),<%=cid%>_requestObject);
}
}
}
} else {
List<String> <%=cid%>_paths = new java.util.ArrayList<String>(java.util.Arrays.asList(parentPath.split("/")));
<%=cid%>_boxFolder = getBoxFolderRecursively(<%=cid%>_paths, <%=cid%>_boxFolder, <%=cid%>_client);
}
}
<%}%>
} else if (<%=cid%>_boxItem.getType().equals("file")) {
com.box.boxjavalibv2.dao.BoxFile <%=cid%>_boxFile = (com.box.boxjavalibv2.dao.BoxFile) <%=cid%>_boxItem;
<%=cid%>_items.add(<%=cid%>_boxFile);
}
for (com.box.boxjavalibv2.dao.BoxItem <%=cid%>_item : <%=cid%>_items) {
String <%=cid%>_itemPath = getBoxItemPath(<%=cid%>_item);
globalMap.put("<%=cid%>_NAME", <%=cid%>_item.getName());
globalMap.put("<%=cid%>_ID", <%=cid%>_item.getId());
globalMap.put("<%=cid%>_FILE_PATH", <%=cid%>_itemPath);
globalMap.put("<%=cid%>_FILE_DIRECTORY", <%=cid%>_itemPath + <%=cid%>_item.getName());
globalMap.put("<%=cid%>_TYPE",<%=cid%>_item.getType());
globalMap.put("<%=cid%>_LAST_MODIFIED", <%=cid%>_item.getModifiedAt());
globalMap.put("<%=cid%>_SIZE", <%=cid%>_item.getSize());
if (<%=cid%>_boxFile == null && <%=cid%>_boxFolder == null)
{
throw new Exception("<%=cid%> - " + "No file or directory found in " + <%=path%>);
}
else if (<%=cid%>_boxFolder != null)
{
java.util.List<com.box.boxjavalibv2.dao.BoxItem> <%=cid%>_children = new java.util.ArrayList<com.box.boxjavalibv2.dao.BoxItem>();
<%if(includeSubDirectories){%>
<%if(filelistType.equalsIgnoreCase("Directories")){%>
<%=cid%>_children = getChildrenRecursively(<%=cid%>_client, <%=cid%>_boxFolder, <%=cid%>_children, true, false);
<%} else if (filelistType.equalsIgnoreCase("Files")){%>
<%=cid%>_children = getChildrenRecursively(<%=cid%>_client, <%=cid%>_boxFolder, <%=cid%>_children, false, true);
<%} else {%>
<%=cid%>_children = getChildrenRecursively(<%=cid%>_client, <%=cid%>_boxFolder, <%=cid%>_children, true, true);
<%}%>
<%}%>
<%else{%>
for (com.box.boxjavalibv2.dao.BoxTypedObject obj : <%=cid%>_boxFolder.getItemCollection().getEntries())
{
<%if(filelistType.equalsIgnoreCase("Directories")){%>
if(obj.getType().equals("folder"))
{
com.box.boxjavalibv2.dao.BoxFolder folder = <%=cid%>_client.getFoldersManager().getFolder(obj.getId(), null);
<%=cid%>_children.add(folder);
}
<%} else if (filelistType.equalsIgnoreCase("Files")){%>
if (obj.getType().equals("file"))
{
com.box.boxjavalibv2.dao.BoxFile file = <%=cid%>_client.getFilesManager().getFile(obj.getId(), null);
<%=cid%>_children.add(file);
}
<%} else {%>
if(obj.getType().equals("folder"))
{
com.box.boxjavalibv2.dao.BoxFolder folder = <%=cid%>_client.getFoldersManager().getFolder(obj.getId(), null);
<%=cid%>_children.add(folder);
}
else if (obj.getType().equals("file"))
{
com.box.boxjavalibv2.dao.BoxFile file = <%=cid%>_client.getFilesManager().getFile(obj.getId(), null);
<%=cid%>_children.add(file);
}
<%}%>
}
<%}%>
for (com.box.boxjavalibv2.dao.BoxItem <%=cid%>_item : <%=cid%>_children)
{
String <%=cid%>_itemPath = getBoxItemPath(<%=cid%>_item);
globalMap.put("<%=cid%>_NAME", <%=cid%>_item.getName());
globalMap.put("<%=cid%>_ID", <%=cid%>_item.getId());
globalMap.put("<%=cid%>_FILE_PATH", <%=cid%>_itemPath);
globalMap.put("<%=cid%>_FILE_DIRECTORY", <%=cid%>_itemPath + <%=cid%>_item.getName());
globalMap.put("<%=cid%>_TYPE",<%=cid%>_item.getType());
globalMap.put("<%=cid%>_LAST_MODIFIED", <%=cid%>_item.getModifiedAt());
globalMap.put("<%=cid%>_SIZE", <%=cid%>_item.getSize());

View File

@@ -16,6 +16,7 @@
%>
}
}
//globalMap.put("<%=cid%>_NB_FILE", NB_FILE<%=cid%>);

View File

@@ -898,8 +898,8 @@
<IMPORT NAME="Driver-VERTICA_7.0.1" MODULE="vertica-jdbc-7.0.1-0.jar" MVN="mvn:org.talend.libraries/vertica-jdbc-7.0.1-0/6.0.0" REQUIRED_IF="(DBTYPE=='VERTICA') and (USE_EXISTING_CONNECTION == 'false') and (DB_VERTICA_VERSION=='VERTICA_7_0_X')" />
<IMPORT NAME="Driver-VERTICA_7.1.2" MODULE="vertica-jdbc-7.1.2-0.jar" MVN="mvn:org.talend.libraries/vertica-jdbc-7.1.2-0/6.0.0" REQUIRED_IF="(DBTYPE=='VERTICA') and (USE_EXISTING_CONNECTION == 'false') and (DB_VERTICA_VERSION=='VERTICA_7_1_X')" />
<IMPORT NAME="Driver-EXASolution" MODULE="exajdbc-6.0.9302.jar" MVN="mvn:org.talend.libraries/exajdbc-6.0.9302/6.3.0" REQUIRED_IF="(DBTYPE=='EXASOL') and (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="Driver-Redshift" MODULE="redshift-jdbc42-no-awssdk-1.2.32.1056.jar" MVN="mvn:com.amazon.redshift/redshift-jdbc42-no-awssdk/1.2.32.1056" REQUIRED_IF="(DBTYPE=='REDSHIFT') and (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="Driver-Snowflake" MODULE="snowflake-jdbc-3.11.1.jar" MVN="mvn:net.snowflake/snowflake-jdbc/3.11.1" REQUIRED_IF="(DBTYPE=='SNOWFLAKE') and (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="Driver-Redshift" MODULE="redshift-jdbc42-no-awssdk-1.2.20.1043.jar" MVN="mvn:com.amazon.redshift/redshift-jdbc42-no-awssdk/1.2.20.1043" REQUIRED_IF="(DBTYPE=='REDSHIFT') and (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="Driver-Snowflake" MODULE="snowflake-jdbc-3.5.3.jar" MVN="mvn:net.snowflake/snowflake-jdbc/3.5.3" REQUIRED_IF="(DBTYPE=='SNOWFLAKE') and (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="Driver-VERTICA_9.0" MODULE="vertica-jdbc-9.0.0-0.jar" MVN="mvn:org.talend.libraries/vertica-jdbc-9.0.0-0/6.0.0" REQUIRED_IF="(DBTYPE=='VERTICA') and (USE_EXISTING_CONNECTION == 'false') and (DB_VERTICA_VERSION=='VERTICA_9_0')"/>
</IMPORTS>
</CODEGENERATION>

View File

@@ -2103,7 +2103,7 @@ class SnowflaketManager extends Manager {
StringBuilder javaCode = new StringBuilder();
if(useExistingConnection) {
javaCode.append(retrieveTable());
javaCode.append("java.sql.Connection conn_" + cid + " = (java.sql.Connection)globalMap.get(\"" + connection + "_connection\");");
javaCode.append("java.sql.Connection conn_" + cid + " = (java.sql.Connection)globalMap.get(\"" + connection + "_Connection\");");
} else {
javaCode.append(super.getConnection());
}

View File

@@ -45,8 +45,8 @@ String xmlfile = ElementParameterParser.getValue(node, "__XMLFILE__");
java.lang.StringBuilder sb<%=cid %>=new java.lang.StringBuilder("");
try{
br<%=cid %> = new java.io.BufferedReader(new UnicodeReader(url<%=cid %>.openStream(), null));
br<%=cid %> = new java.io.BufferedReader(new java.io.InputStreamReader(url<%=cid %>.openStream()));
char[] buffer<%=cid %> = new char[1024];
int length<%=cid %> = -1;
while ((length<%=cid %> = br<%=cid %>.read(buffer<%=cid %>)) != -1)
@@ -66,9 +66,7 @@ String xmlfile = ElementParameterParser.getValue(node, "__XMLFILE__");
dbf<%=cid %> = javax.xml.parsers.DocumentBuilderFactory.newInstance();
dbf<%=cid %>.setValidating(false);
db<%=cid %> = dbf<%=cid %>.newDocumentBuilder();
org.w3c.dom.Document doc<%=cid %> = db<%=cid %>.parse(
new java.io.ByteArrayInputStream(sb<%=cid%>.toString().getBytes("UTF-8")));
org.w3c.dom.Document doc<%=cid %> = db<%=cid %>.parse(new java.io.StringBufferInputStream(sb<%=cid%>.toString()));
String rootnode<%=cid %>=doc<%=cid %>.getDocumentElement().getNodeName();
String encoding=null;
@@ -91,8 +89,8 @@ String xmlfile = ElementParameterParser.getValue(node, "__XMLFILE__");
int offsetRoot<%=cid %>=sb<%=cid %>.indexOf("<"+rootnode<%=cid %>);
sb<%=cid%>.replace(0, offsetRoot<%=cid %>, reference<%=cid %>);
is<%=cid %> = new java.io.ByteArrayInputStream(sb<%=cid%>.toString().getBytes("UTF-8"));
is<%=cid %>=new java.io.StringBufferInputStream(sb<%=cid %>.toString());
class MyHandler<%=cid %> extends org.xml.sax.helpers.DefaultHandler{
String errorMessage = null;
public void error(org.xml.sax.SAXParseException e) throws org.xml.sax.SAXException {

View File

@@ -40,8 +40,8 @@
</ADVANCED_PARAMETERS>
<CODEGENERATION>
<IMPORTS>
<IMPORT MODULE="janet-java-1.2.jar" MVN="mvn:org.talend.libraries/janet-java-1.2/6.0.0" NAME="janet-java-1.2"
REQUIRED="true" />
<IMPORT MODULE="janet-java-1.1.jar" MVN="mvn:org.talend.libraries/janet-java-1.1/6.0.0" NAME="janet-java-1.1"
REQUIRED="true" />
</IMPORTS>
</CODEGENERATION>
<RETURNS>

View File

@@ -94,7 +94,7 @@
</ADVANCED_PARAMETERS>
<CODEGENERATION>
<IMPORTS>
<IMPORT MODULE="janet-java-1.2.jar" MVN="mvn:org.talend.libraries/janet-java-1.2/6.0.0" NAME="janet-java-1.2"
<IMPORT MODULE="janet-java-1.1.jar" MVN="mvn:org.talend.libraries/janet-java-1.1/6.0.0" NAME="janet-java-1.1"
REQUIRED="true" />
</IMPORTS>
</CODEGENERATION>

View File

@@ -25,7 +25,7 @@
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
org.talend.designer.dbmap.language.mssql.MssqlGenerationManager gm = new org.talend.designer.dbmap.language.mssql.MssqlGenerationManager();
org.talend.designer.dbmap.language.generation.GenericDbGenerationManager gm = new org.talend.designer.dbmap.language.generation.GenericDbGenerationManager();
String uniqueNameComponent = null;
IDbLanguage currentLanguage = gm.getLanguage();

View File

@@ -305,54 +305,48 @@ for(Column colStmt:stmtStructure){
if(("INSERT").equals(dataAction)){
%>
String insertQuery_<%=cid %> = "INSERT INTO "+tableName_<%=cid%>+"(<%=insertColName.toString()%>) ("+select_query_<%=cid %>+")";
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(insertQuery_<%=cid %>);
String insertQuery_<%=cid %> = "INSERT INTO "+tableName_<%=cid%>+"(<%=insertColName.toString()%>) ("+select_query_<%=cid %>+")";
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(insertQuery_<%=cid %>);
<%
}else if (("UPDATE").equals(dataAction)){
if(useUpdateStatement){
if(useUpdateStatement){
%>
String updateQuery_<%=cid %> = select_query_<%=cid %>;
String updateQuery_<%=cid %> = select_query_<%=cid %>
<%
}else{
}else{
%>
String updateQuery_<%=cid %> = "UPDATE "+tableName_<%=cid%>+" SET <%=updateSetStmt.toString()%> ";
String updateQuery_<%=cid %> = "UPDATE "+tableName_<%=cid%>+" SET <%=updateSetStmt.toString()%> "
<%
}
if(CodeGenerationUtils.hasAlphaNumericCharacter(whereClause)) {
if(useUpdateStatement){
}
%>
if(select_query_<%=cid %>.indexOf("WHERE")==-1){
updateQuery_<%=cid %> +=" WHERE ";
}else{
updateQuery_<%=cid %> +=" AND ";
}
updateQuery_<%=cid %>+= <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>;
<%
}else{
%>
updateQuery_<%=cid %> += " WHERE " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>;
<%
}
}
%>
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(updateQuery_<%=cid %>);
<%
}else if (("DELETE").equals(dataAction)){
%>
String deleteQuery_<%=cid %> = "DELETE FROM "+ tableName_<%=cid%>+" WHERE EXISTS ("+select_query_<%=cid %>+") "
<%
if(CodeGenerationUtils.hasAlphaNumericCharacter(whereClause)) {
%>
+" AND " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>
+" WHERE " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>
<%
}
%>
;
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(deleteQuery_<%=cid %>);
<%
}
%>
;
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(updateQuery_<%=cid %>);
<%
}else if (("DELETE").equals(dataAction)){
%>
String deleteQuery_<%=cid %> = "DELETE FROM "+ tableName_<%=cid%>+" WHERE EXISTS ("+select_query_<%=cid %>+") "
<%
if(CodeGenerationUtils.hasAlphaNumericCharacter(whereClause)) {
%>
+" AND " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>
<%
}
%>
;
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(deleteQuery_<%=cid %>);
<%
}
%>
int nb_line_<%=cid%> = 0;
int nb_line_updated_<%=cid%> = 0;
int nb_line_inserted_<%=cid%> = 0;

View File

@@ -33,4 +33,4 @@ USE_DIFFERENT_TABLE.NAME=Use different table name
DIFFERENT_TABLE_NAME.NAME=Table name
IS_TABLE_NAME_VARIABLE.NAME=Table name from connection name is variable
USE_UPDATE_STATEMENT.NAME=Use update statement without subqueries
USE_UPDATE_STATEMENT.NAME=Use update statement

View File

@@ -181,17 +181,6 @@
</PARAMETER>
</PARAMETERS>
<ADVANCED_PARAMETERS>
<PARAMETER
NAME="PROPERTIES"
FIELD="TEXT"
NUM_ROW="1"
REPOSITORY_VALUE="PROPERTIES_STRING"
SHOW_IF="(#LINK@CONNECTOR.OUT.TABLE_REF == '0') AND (USE_EXISTING_CONNECTION == 'false')">
<DEFAULT>""</DEFAULT>
</PARAMETER>
</ADVANCED_PARAMETERS>
<CODEGENERATION>
<IMPORTS>

View File

@@ -48,5 +48,4 @@ DB_VERSION.ITEM.MYSQL_5=Mysql 5
DB_VERSION.ITEM.MYSQL_8=Mysql 8
DB_VERSION.ITEM.MARIADB=MariaDB
PROPERTIES.NAME=Additional JDBC Parameters
CONNECTION.NAME=Component List

View File

@@ -52,7 +52,6 @@ imports="
String dbuser = null;
String dbVersion = "";
boolean useExistingConn = false;
String dbproperties = null;
if(previousNode != null) {
dbhost = ElementParameterParser.getValue(previousNode, "__HOST__");
dbport = ElementParameterParser.getValue(previousNode, "__PORT__");
@@ -60,7 +59,6 @@ imports="
dbuser = ElementParameterParser.getValue(previousNode, "__USER__");
useExistingConn = ("true").equals(ElementParameterParser.getValue(previousNode, "__USE_EXISTING_CONNECTION__"));
dbVersion = ElementParameterParser.getValue(previousNode, "__DB_VERSION__");
dbproperties = ElementParameterParser.getValue(previousNode, "__PROPERTIES__");
}
String differenttable = ElementParameterParser.getValue(node, "__DIFFERENT_TABLE_NAME__");
@@ -98,11 +96,7 @@ imports="
%>
String driverClass_<%=cid%> = "<%=drverClass%>";
java.lang.Class.forName(driverClass_<%=cid%>);
<%if(dbproperties == null || ("\"\"").equals(dbproperties) || ("").equals(dbproperties)) {%>
String url_<%=cid %> = "<%=jdbcURL%>://"+<%=dbhost%>+":"+<%=dbport%>+"/"+<%=dbname%>;
<%} else {%>
String url_<%=cid %> = "<%=jdbcURL%>://"+<%=dbhost%>+":"+<%=dbport%>+"/"+<%=dbname%>+"?"+<%=dbproperties%>;
<%}%>
String dbUser_<%=cid%> = <%=dbuser%>;
<%

View File

@@ -140,15 +140,7 @@
</PARAMETERS>
<ADVANCED_PARAMETERS>
<PARAMETER
NAME="USE_UPDATE_STATEMENT"
FIELD="CHECK"
NUM_ROW="10"
SHOW_IF="DATA_ACTION=='UPDATE'"
>
<DEFAULT>false</DEFAULT>
</PARAMETER>
<ADVANCED_PARAMETERS>
</ADVANCED_PARAMETERS>
<CODEGENERATION>

View File

@@ -35,7 +35,6 @@ skeleton="../templates/db_output_bulk.skeleton"
String differenttable = ElementParameterParser.getValue(node, "__DIFFERENT_TABLE_NAME__");
boolean useDifferentTable = "true".equals(ElementParameterParser.getValue(node, "__USE_DIFFERENT_TABLE__"));
boolean isTableNameVariable="true".equals(ElementParameterParser.getValue(node, "__IS_TABLE_NAME_VARIABLE__"));
boolean useUpdateStatement="true".equals(ElementParameterParser.getValue(node, "__USE_UPDATE_STATEMENT__"));
%>
String select_query_<%=cid %> = null;
String tableName_<%=cid%> = null;
@@ -94,7 +93,7 @@ skeleton="../templates/db_output_bulk.skeleton"
String username = "username_" + connection;
String connectionKey=null;
if("tSnowflakeConnection".equals(connNode.getComponent().getName())){
connectionKey=connection+"_connection";
connectionKey=connection+"_Connection";
}else{
connectionKey=conn;
}
@@ -157,36 +156,21 @@ skeleton="../templates/db_output_bulk.skeleton"
}
if(("INSERT").equals(dataAction)){
%>
String insertQuery_<%=cid %> = "INSERT INTO "+tableName_<%=cid%>+"(<%=insertColName.toString()%>) ("+select_query_<%=cid %>+")";
<%
} else if (("UPDATE").equals(dataAction)){
if(useUpdateStatement){
%>
String updateQuery_<%=cid %> = select_query_<%=cid %>;
<%
if(CodeGenerationUtils.hasAlphaNumericCharacter(whereClause)) {
%>
if (updateQuery_<%=cid %>.toUpperCase().contains(" WHERE ")) {
updateQuery_<%=cid %> += " AND (" + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%> + ")";
} else {
updateQuery_<%=cid %> += " WHERE " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>;
}
<%
}
} else {
%>
String updateQuery_<%=cid %> = "UPDATE "+tableName_<%=cid%>+" SET <%=updateSetStmt.toString()%> "
<%
if(CodeGenerationUtils.hasAlphaNumericCharacter(whereClause)) {
%>
+" WHERE " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>
<%
}
%>
;
%>
String insertQuery_<%=cid %> = "INSERT INTO "+tableName_<%=cid%>+"(<%=insertColName.toString()%>) ("+select_query_<%=cid %>+")";
<%
} else if (("UPDATE").equals(dataAction)){
%>
String updateQuery_<%=cid %> = "UPDATE "+tableName_<%=cid%>+" SET <%=updateSetStmt.toString()%> "
<%
if(CodeGenerationUtils.hasAlphaNumericCharacter(whereClause)) {
%>
+" WHERE " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>
<%
}
%>
;
<%
}
} else if (("DELETE").equals(dataAction)){
%>
String deleteQuery_<%=cid %> = "DELETE FROM "+ tableName_<%=cid%>+" WHERE EXISTS ("+select_query_<%=cid %>+") "

View File

@@ -40,6 +40,4 @@ TABLE_ACTION.ITEM.CREATE_IF_NOT_EXISTS=Create table if not exists
TABLE_ACTION.ITEM.DROP_IF_EXISTS_AND_CREATE=Drop table if exists and create
TABLE_ACTION.ITEM.DROP_CREATE=Drop and create table
TABLE_ACTION.ITEM.NONE=None
TABLE_ACTION.NAME=Action on table
USE_UPDATE_STATEMENT.NAME=Use update statement without subqueries
TABLE_ACTION.NAME=Action on table

View File

@@ -107,7 +107,7 @@ imports="
}
else{
%>
String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName=" + <%=dbname %> + ";" + <%=dbproperties%>;
String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName" + <%=dbname %> + ";" + <%=dbproperties%>;
<%
}
}

View File

@@ -178,7 +178,7 @@
<IMPORTS>
<IMPORT NAME="Java_DOM4J1.6" MODULE="dom4j-1.6.1.jar" MVN="mvn:dom4j/dom4j/1.6.1" UrlPath="platform:/plugin/org.talend.libraries.dom4j-jaxen/lib/dom4j-1.6.1.jar" REQUIRED_IF="READ_BY == 'XPATH'" BundleID="" />
<IMPORT NAME="Java_JAXEN1.1" MODULE="jaxen-1.1.1.jar" MVN="mvn:org.talend.libraries/jaxen-1.1.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.dom4j-jaxen/lib/jaxen-1.1.1.jar" REQUIRED_IF="READ_BY == 'XPATH'" BundleID="" />
<IMPORT NAME="json-lib" MODULE="json-lib-2.4.5-talend.jar" MVN="mvn:net.sf.json-lib/json-lib/2.4.5-talend" REQUIRED_IF="READ_BY == 'XPATH'" />
<IMPORT NAME="json-lib" MODULE="json-lib-2.4.2-talend.jar" MVN="mvn:net.sf.json-lib/json-lib/2.4.2-talend" REQUIRED_IF="READ_BY == 'XPATH'" />
<IMPORT NAME="commons_lang" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED_IF="READ_BY == 'XPATH'" />
<IMPORT NAME="commons_logging" MODULE="commons-logging-1.1.1.jar" MVN="mvn:org.talend.libraries/commons-logging-1.1.1/6.0.0" UrlPath="platform:/base/plugins/org.apache.commons.logging_1.1.1.v201101211721.jar" REQUIRED_IF="READ_BY == 'XPATH'" />
<IMPORT NAME="ezmorph" MODULE="ezmorph-1.0.6.jar" MVN="mvn:org.talend.libraries/ezmorph-1.0.6/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.jackson/lib/ezmorph-1.0.6.jar" REQUIRED_IF="READ_BY == 'XPATH'" />

View File

@@ -149,7 +149,7 @@
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Talen File Enhanced"
MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar" REQUIRED="true" />
MODULE="talend_file_enhanced_20070724.jar" MVN="mvn:org.talend.libraries/talend_file_enhanced_20070724/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced_20070724.jar" REQUIRED="true" />
</IMPORTS>
</CODEGENERATION>

View File

@@ -17,16 +17,20 @@
if (connObj != null) {
try {
<% if (sftp) { %>
<% if (sftp == false && ftps == false) { %>
com.enterprisedt.net.ftp.FTPClient conn = (com.enterprisedt.net.ftp.FTPClient) connObj;
conn.executeCommand("quit");
<% } else if (sftp) { %>
com.jcraft.jsch.ChannelSftp channel = (com.jcraft.jsch.ChannelSftp) connObj;
com.jcraft.jsch.Session session = channel.getSession();
channel.disconnect();
session.disconnect();
<% } else { %>
org.apache.commons.net.ftp.FTPClient conn = (org.apache.commons.net.ftp.FTPClient) connObj;
<% } else if (ftps) { %>
org.apache.commons.net.ftp.FTPSClient conn = (org.apache.commons.net.ftp.FTPSClient) connObj;
conn.disconnect();
<% }
%>
<% } else { %>
System.err.println("Invalid tFTPConnection configuration.");
<% } %>
} catch (Exception e) {
globalMap.put("<%=cid%>_ERROR_MESSAGE", e.getMessage());

View File

@@ -15,13 +15,16 @@
%>
Object connObj = globalMap.get("conn_<%=connection%>");
if (connObj != null) {
<% if (sftp) { %>
<% if (sftp == false && ftps == false) { %>
com.enterprisedt.net.ftp.FTPClient conn = (com.enterprisedt.net.ftp.FTPClient) connObj;
conn.executeCommand("quit");
<% } else if (sftp) { %>
com.jcraft.jsch.ChannelSftp channel = (com.jcraft.jsch.ChannelSftp) connObj;
com.jcraft.jsch.Session session = channel.getSession();
channel.disconnect();
session.disconnect();
<% } else { %>
org.apache.commons.net.ftp.FTPClient conn = (org.apache.commons.net.ftp.FTPClient) connObj;
<% } else if (ftps) { %>
org.apache.commons.net.ftp.FTPSClient conn = (org.apache.commons.net.ftp.FTPSClient) connObj;
conn.disconnect();
<% } %>

View File

@@ -0,0 +1,88 @@
<%@ jet
%>
boolean retry_<%=cid%> = false;
int retry_count_<%=cid%> = 0;
int retry_max_<%=cid%> = 5;
com.jcraft.jsch.Session session_<%=cid%> = null;
com.jcraft.jsch.Channel channel_<%=cid%> = null;
do {
retry_<%=cid%> = false;
com.jcraft.jsch.JSch jsch_<%=cid%> = new com.jcraft.jsch.JSch();
<%if (("PUBLICKEY").equals(authMethod)){%>
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - SFTP authentication using a public key.");
log.debug("<%=cid%> - Private key: '" + <%=privateKey%> + "'.");
<%}%>
jsch_<%=cid%>.addIdentity(<%=privateKey %>, defaultUserInfo_<%=cid%>.getPassphrase());
<%}%>
session_<%=cid%> = jsch_<%=cid%>.getSession(<%=user%>, <%=host%>, <%=port%>);
session_<%=cid%>.setConfig("PreferredAuthentications", "publickey,password,keyboard-interactive,gssapi-with-mic");
<%if (("PASSWORD").equals(authMethod)) {%>
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - SFTP authentication using a password.");
<%}%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
session_<%=cid%>.setPassword(decryptedPassword_<%=cid%>);
<%}%>
session_<%=cid%>.setUserInfo(defaultUserInfo_<%=cid%>);
<%if (!useProxy) {%>
if(("true").equals(System.getProperty("http.proxySet")) ){
com.jcraft.jsch.ProxyHTTP proxy_<%=cid%> = new com.jcraft.jsch.ProxyHTTP(System.getProperty("http.proxyHost"),Integer.parseInt(System.getProperty("http.proxyPort")));
if(!"".equals(System.getProperty("http.proxyUser"))){
proxy_<%=cid%>.setUserPasswd(System.getProperty("http.proxyUser"),System.getProperty("http.proxyPassword"));
}
session_<%=cid%>.setProxy(proxy_<%=cid%>);
}
<%}%>
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user%> + "'.");
<%}%>
channel_<%=cid%> = null;
try {
session_<%=cid%>.connect();
channel_<%=cid%> = session_<%=cid%>.openChannel("sftp");
channel_<%=cid%>.connect();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
} catch (com.jcraft.jsch.JSchException e_<%=cid%>) {
try {
if(channel_<%=cid%>!=null) {
channel_<%=cid%>.disconnect();
}
if(session_<%=cid%>!=null) {
session_<%=cid%>.disconnect();
}
} catch(java.lang.Exception ce_<%=cid%>) {
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - close sftp connection failed : " + ce_<%=cid%>.getClass() + " : " + ce_<%=cid%>.getMessage());
<%}%>
}
String message_<%=cid%> = new TalendException(null, null, null).getExceptionCauseMessage(e_<%=cid%>);
if(message_<%=cid%>.contains("Signature length not correct") || message_<%=cid%>.contains("connection is closed by foreign host")) {
retry_<%=cid%> = true;
retry_count_<%=cid%>++;
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - connect: Signature length not correct or connection is closed by foreign host, so retry, retry time : " + retry_count_<%=cid%>);
<%}%>
} else {
throw e_<%=cid%>;
}
}
} while(retry_<%=cid%> && (retry_count_<%=cid%> < retry_max_<%=cid%>));
com.jcraft.jsch.ChannelSftp c_<%=cid%> = (com.jcraft.jsch.ChannelSftp)channel_<%=cid%>;

View File

@@ -28,7 +28,7 @@ String authMethod = ElementParameterParser.getValue(node, "__AUTH_METHOD__");
String privateKey = ElementParameterParser.getValue(node, "__PRIVATEKEY__");
boolean ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
String encoding = ElementParameterParser.getValue(node, "__ENCODING__");
String sEncoding = ElementParameterParser.getValue(node, "__ENCODING__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
String protectionLevel = ElementParameterParser.getValue(node, "__FTPS_PROT__");
String protectionBufferSize = ElementParameterParser.getValue(node, "__FTPS_PROTECTION_BUFF_SIZE__");
@@ -60,22 +60,42 @@ if (ElementParameterParser.canEncrypt(node, passwordFieldName)) {
});
<%
}
%>
<%
if (!sftp && !ftps) { // *** ftp *** //
%>
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
<%
passwordFieldName = "__PASS__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
globalMap.put("conn_<%=cid%>",ftp_<%=cid %>);
if (!sftp && !ftps) { // *** ftp *** //
if(isLog4jEnabled){%>
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
<%}%>
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
ftp_<%=cid %>.setRemoteHost(<%=host %>);
ftp_<%=cid %>.setRemotePort(<%=port %>);
<%if (("ACTIVE").equals(connectMode)) {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
<%} else {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
<%}%>
ftp_<%=cid %>.setControlEncoding(<%=sEncoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect();
<%
passwordFieldName = "__PASS__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
globalMap.put("conn_<%=cid%>",ftp_<%=cid %>);
<%
} else if (!ftps) { // *** sftp *** //
%>
class MyUserInfo implements com.jcraft.jsch.UserInfo, com.jcraft.jsch.UIKeyboardInteractive {
<%
passwordFieldName = "__PASSPHRASE__";
if (ElementParameterParser.canEncrypt(node, passwordFieldName)) {
@@ -118,20 +138,99 @@ if (!sftp && !ftps) { // *** ftp *** //
passwordFieldName = "__PASS__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
<%@ include file="sftp_connect.javajet"%>
<%if(use_encoding) {%>
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
c_<%=cid%>.setFilenameEncoding(<%=sEncoding%>);
<%}%>
globalMap.put("conn_<%=cid%>", c_<%=cid%>);
<%
} else { // *** ftps *** //
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
globalMap.put("conn_<%=cid%>", ftp_<%=cid %>);
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
MyTrust_<%=cid%> myTrust_<%=cid%> = null;
try {
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
if (isHTTPProxyUsed) {
String httpProxyHost = System.getProperty("http.proxyHost");
int httpProxyPort = Integer.getInteger("http.proxyPort");
String httpProxyUser = System.getProperty("http.proxyUser");
String httpProxyPass = System.getProperty("http.proxyPassword");
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
} else {
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
}
ftp_<%=cid %>.setControlEncoding(<%=sEncoding %>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
<%
passwordFieldName = "__PASS__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
if (!isLoginSuccessful_<%=cid %>) {
throw new RuntimeException("Login failed");
}
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
<%
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
%>
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
<%
}
if (protectionLevel != null && !protectionLevel.isEmpty()) {
%>
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
<%
}
%>
} catch (Exception e) {
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
<%}%>
throw e;
}
<%
if ("PASSIVE".equals(connectMode)) {
%>
ftp_<%=cid %>.enterLocalPassiveMode();
<%
if (isLog4jEnabled) {
%>
log.debug("<%=cid%> - Using the passive mode.");
<%
}
}
%>
globalMap.put("conn_<%=cid%>", ftp_<%=cid %>);
<%
}
%>

View File

@@ -185,6 +185,7 @@
REQUIRED="true" NOT_SHOW_IF="(SFTP == 'true') and (USE_ENCODING == 'false')">
<DEFAULT>"ISO-8859-15"</DEFAULT>
</PARAMETER>
</PARAMETERS>
<ADVANCED_PARAMETERS>
@@ -235,10 +236,10 @@
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false')"/>
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true')"/>
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false')" />
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar" MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819" REQUIRED_IF="(FTPS == 'true')" />
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED="true" />
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true')" />
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar" MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819" REQUIRED_IF="(FTPS == 'true')" />
</IMPORTS>
</CODEGENERATION>

View File

@@ -55,8 +55,6 @@ if (("true").equals(useExistingConn)) {
} else {
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
%>
<%
}
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
@@ -131,7 +129,7 @@ int nb_file_<%=cid%> = 0;
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
<%if(use_encoding) {%>
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
@@ -168,70 +166,191 @@ int nb_file_<%=cid%> = 0;
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
<%
} else {
if (ftps) { // *** ftps *** //
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
<%
} else if (ftps) { // *** ftps *** //
if (("false").equals(useExistingConn)) {
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
try {
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
if (isHTTPProxyUsed) {
String httpProxyHost = System.getProperty("http.proxyHost");
int httpProxyPort = Integer.getInteger("http.proxyPort");
String httpProxyUser = System.getProperty("http.proxyUser");
String httpProxyPass = System.getProperty("http.proxyPassword");
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
} else {
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
}
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
if (!isLoginSuccessful_<%=cid %>) {
throw new RuntimeException("Login failed");
}
<%
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
%>
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
<%
}
if (protectionLevel != null && !protectionLevel.isEmpty()) {
%>
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
<%
}
%>
} catch (Exception e) {
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
<%}%>
<%
if (dieOnError) {
%>
throw e;
<%
} else {
%>
e.printStackTrace();
<%
}
%>
}
<%
if ("PASSIVE".equals(connectMode)) {
%>
ftp_<%=cid %>.enterLocalPassiveMode();
<%
if (isLog4jEnabled) {
%>
log.debug("<%=cid%> - Using the passive mode.");
<%
}
}
} else {
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
<%
if (moveToCurrentDir) {
%>
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(<%=remotedir %>);
if (!cwdSuccess_<%=cid %>) {
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
}
<%
}
}
%>
<% /*store real pwd*/ %>
String rootWorkDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<>();
<%
} else {// *** ftp *** //
for (int i=0; i<files.size(); i++) {
Map<String, String> line = files.get(i);
%>
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> =null;
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,"");
list<%=cid %>.add(map<%=cid %><%=i %>);
<%
}
%>
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
<%
} else {// *** ftp *** //
%>
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> =null;
<%if (("true").equals(useExistingConn)) {%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Uses an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
log.info("<%=cid%> - Uses an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ",Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%if(!moveToCurrentDir){%>
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
<%}%>
<%} else {%>
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
ftp_<%=cid %>.setRemoteHost(<%=host %>);
ftp_<%=cid %>.setRemotePort(<%=port %>);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
<%}%>
<%if (("ACTIVE").equals(connectMode)) {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
<%} else {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
<%}%>
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username: '" +<%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect();
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
<%}%>
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
<%} else {
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
<%}%>
<%
}//common code ftp and ftps:
%>
<%
if (("true").equals(useExistingConn) && moveToCurrentDir) {
%>
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(<%=remotedir %>);
<%for (int i = 0; i < files.size(); i++) {
Map<String, String> line = files.get(i);
%>
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,"");
list<%=cid %>.add(map<%=cid %><%=i %>);
<%}%>
String remotedir<%=cid %> = <%=remotedir%>;
ftp_<%=cid %>.chdir(remotedir<%=cid %>);
String root<%=cid %> = ftp_<%=cid %>.pwd();
if (!cwdSuccess_<%=cid %>) {
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
}
<%
}
%>
<% /*store real pwd*/ %>
String rootWorkDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<>();
<%
for (int i=0; i<files.size(); i++) {
Map<String, String> line = files.get(i);
%>
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,"");
list<%=cid %>.add(map<%=cid %><%=i %>);
<%
}
%>
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
<%
}
%>
<%if(isLog4jEnabled){%>
<%if("FILE".equals(targetType)){%>
log.info("<%=cid%> - Deleting file from server.");
<%}else{%>
log.info("<%=cid%> - Deleting directory from server.");
<%}%>
<%}%>
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
<%}%>

View File

@@ -70,14 +70,14 @@
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
}catch(java.io.IOException ioe_<%=cid%>){
}catch(java.net.SocketException se_<%=cid%>){
//ignore failure
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
<%}%>
}
<%
@@ -86,12 +86,18 @@
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
<%
}
}else{
if(!moveToCurrentDir){
%>
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
<%
}
}
}
%>

View File

@@ -177,7 +177,7 @@
<DEFAULT>"."</DEFAULT>
</PARAMETER>
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="31" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="31" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
<DEFAULT>true</DEFAULT>
</PARAMETER>
@@ -292,13 +292,13 @@
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="jakarta-oro" MODULE="jakarta-oro-2.0.8.jar" MVN="mvn:org.talend.libraries/jakarta-oro-2.0.8/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache/lib/jakarta-oro-2.0.8.jar" REQUIRED="true" />
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
</IMPORTS>
</CODEGENERATION>
<RETURNS>

View File

@@ -95,7 +95,7 @@
<%
}else { // *** ftp, ftps *** //
}else if (ftps) { // *** ftps *** //
%>
for (String key<%=cid %> : keySet<%=cid %>) {
<% /*prepare filemask*/ %>
@@ -241,6 +241,73 @@
<% /*return to root dir for next filemask*/ %>
ftp_<%=cid %>.changeWorkingDirectory(rootWorkDir_<%=cid %>);
<%
}else {// *** ftp *** //
%>
try {
for (String key<%=cid %> : keySet<%=cid %>) {
String filemask<%=cid %> = key<%=cid %>;
String dir<%=cid %> = null;
<%if(!usePerl5) {//perl5 mode not support windows(\) path separator at the mask string%>
String mask<%=cid %> = filemask<%=cid %>.replaceAll("\\\\", "/") ;
<%}else{%>
String mask<%=cid %> = filemask<%=cid %>;
<%}%>
int i<%=cid %> = mask<%=cid %>.lastIndexOf('/');
if (i<%=cid %>!=-1) {
dir<%=cid %> = mask<%=cid %>.substring(0, i<%=cid %>);
mask<%=cid %> = mask<%=cid %>.substring(i<%=cid %>+1);
}
if (dir<%=cid %>!=null && !"".equals(dir<%=cid %>)) ftp_<%=cid %>.chdir(dir<%=cid %>);
<%if(!usePerl5) {%>
mask<%=cid %> = org.apache.oro.text.GlobCompiler.globToPerl5(mask<%=cid %>.toCharArray(), org.apache.oro.text.GlobCompiler.DEFAULT_MASK);
<%}%>
String[] fileName_<%=cid%> = null;
boolean dirFailed_<%=cid%> = false;
try{
fileName_<%=cid%> = ftp_<%=cid%>.dir(null,false);
}catch(com.enterprisedt.net.ftp.FTPException e_<%=cid%>){
dirFailed_<%=cid%> = true;
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - " + e_<%=cid%>.getMessage());
<%}%>
}
if((fileName_<%=cid%> == null) || dirFailed_<%=cid%> || (fileName_<%=cid%>.length <= 0)){
com.enterprisedt.net.ftp.FTPFile[] fileList_<%=cid%> = ftp_<%=cid%>.dirDetails(null);
fileName_<%=cid%> = new String[fileList_<%=cid%>.length];
for (int i_<%=cid%> = 0 ; i_<%=cid%> < fileList_<%=cid%>.length ; i_<%=cid%>++){
fileName_<%=cid%>[i_<%=cid%>] = fileList_<%=cid%>[i_<%=cid%>].getName();
}
}
for (String file_<%=cid%> : fileName_<%=cid%>) {
if (file_<%=cid%>.matches(mask<%=cid %>)) {
<%
if("FILE".equals(targetType)){
%>
ftp_<%=cid %>.delete(file_<%=cid%>);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - File '" + file_<%=cid%> + "' was deleted from the remote directory " + <%=remotedir %> + ".");
<%}%>
<%
}else{
%>
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Directory '" + file_<%=cid%> + "' was deleted from the remote directory " + <%=remotedir %> + ".");
<%}%>
ftp_<%=cid %>.rmdir(file_<%=cid%>);
<%
}
%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File deleted.");
nb_file_<%=cid%>++;
}
}
}
}catch(com.enterprisedt.net.ftp.FTPException e_<%=cid %>){
globalMap.put("<%=cid %>_CURRENT_STATUS", "Deleting file action error");
throw e_<%=cid %>;
}
ftp_<%=cid %>.chdir(root<%=cid %>);
<%
}
%>

View File

@@ -57,8 +57,6 @@ if (("true").equals(useExistingConn)) {
} else {
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
%>
<%
}
String passwordFieldName = "";
@@ -136,7 +134,7 @@ if (sftp) { // *** sftp *** //
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
<%if(use_encoding) {%>
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
@@ -155,38 +153,134 @@ if (sftp) { // *** sftp *** //
}
<%
}
} else { // *** ftp+s *** //
if (ftps) {
} else if (ftps) { // *** ftps *** //
if (("false").equals(useExistingConn)) {
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
try {
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
if (isHTTPProxyUsed) {
String httpProxyHost = System.getProperty("http.proxyHost");
int httpProxyPort = Integer.getInteger("http.proxyPort");
String httpProxyUser = System.getProperty("http.proxyUser");
String httpProxyPass = System.getProperty("http.proxyPassword");
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
} else {
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
}
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
<%
if (("false").equals(useExistingConn)) {
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
if (!isLoginSuccessful_<%=cid %>) {
throw new RuntimeException("Login failed");
}
<%
} else {
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
<%
}
} else { // *** ftp *** //
}
if (protectionLevel != null && !protectionLevel.isEmpty()) {
%>
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
<%
if (("true").equals(useExistingConn)) {%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%} else {
passwordFieldName = "__PASSWORD__";
}
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
} catch (Exception e) {
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
<%}%>
throw e;
}
<%
}
if ("PASSIVE".equals(connectMode)) {
%>
ftp_<%=cid %>.enterLocalPassiveMode();
<%
if (isLog4jEnabled) {
%>
log.debug("<%=cid%> - Using the passive mode.");
<%
}
}
} else {
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
<%
}
} else { // *** ftp *** //
%>
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> = null;
<% if (("true").equals(useExistingConn)) {%>
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%} else {%>
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
ftp_<%=cid %>.setRemoteHost(<%=host %>);
ftp_<%=cid %>.setRemotePort(<%=port %>);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
<%}%>
<%if (("ACTIVE").equals(connectMode)) {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
<%} else {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
<%}%>
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username: '" +<%=username %>+ "'.");
<%}%>
ftp_<%=cid %>.connect();
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
ftp_<%=cid %>.login(<%=username %>, decryptedPassword_<%=cid%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host%> + "' has succeeded.");
<%}%>
<%
}
}%>

View File

@@ -56,35 +56,41 @@ if (sftp) { // *** sftp *** //
<%
}
} else { // *** ftp *** //
if (!("true").equals(useExistingConn)) {
if (("true").equals(ignoreFailureAtQuit)) {
%>
try {
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
} catch(java.io.IOException ioe_<%=cid%>) {
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
<%}%>
//ignore failure
}
<%
} else {
if (!("true").equals(useExistingConn)) {
if (("true").equals(ignoreFailureAtQuit)) {
%>
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
try {
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
} catch(java.net.SocketException se_<%=cid%>) {
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
<%}%>
//ignore failure
}
<%
} else {
%>
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
<%
}
} else {
if (moveToCurrentDir) {
%>
ftp_<%=cid %>.chdir(<%=remoteDir %>);
<%
}
}
}
}
}
%>

View File

@@ -170,7 +170,7 @@ COMPATIBILITY="ALL"
<DEFAULT>"."</DEFAULT>
</PARAMETER>
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="61" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="61" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
<DEFAULT>true</DEFAULT>
</PARAMETER>
@@ -280,12 +280,12 @@ COMPATIBILITY="ALL"
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
</IMPORTS>
</CODEGENERATION>

View File

@@ -70,7 +70,7 @@ if (sftp) { // *** sftp *** //
}
globalMap.put("<%=cid %>_DIRECTORYNAME", <%=directoryname %>);
<%}
} else { // *** ftp+s *** //
} else if (ftps) { // *** ftps *** //
%>
//change working dir and save root
String remoteDir_<%=cid %> = (<%=remoteDir %>).replaceAll("\\\\","/");
@@ -155,5 +155,35 @@ if (sftp) { // *** sftp *** //
ftp_<%=cid %>.changeWorkingDirectory(remoteDir_<%=cid %>);
<%
}
}
%>
} else { // *** ftp *** //
if (isFileType){%>
String remoteDir_<%=cid %> = (<%=remoteDir %>).replaceAll("\\\\","/");
if (ftp_<%=cid %>.exists<%="File" %>(remoteDir_<%=cid %> + "/" + <%=filename %>)) {
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - '" + <%=filename%> + "' exists in the remote directory '"+ <%=remoteDir %> + "' .");
<%}%>
globalMap.put("<%=cid %>_EXISTS", true);
} else {
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - '" + <%=filename%> + "' doesn't exist in the remote directory '"+ <%=remoteDir %> + "' .");
<%}%>
globalMap.put("<%=cid %>_EXISTS", false);
}
globalMap.put("<%=cid %>_FILENAME", <%=filename %>);
<%} else {%>
String remoteDir_<%=cid %> = (<%=remoteDir %>).replaceAll("\\\\","/");
String directoryname_<%=cid %> = (<%=directoryname %>).replaceAll("\\\\","/");
if (ftp_<%=cid %>.exists<%="Directory" %>(remoteDir_<%=cid %> + "/" + directoryname_<%=cid %>)) {
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - '" + <%=directoryname %> + "' exists in the remote directory '"+ <%=remoteDir %> + "' .");
<%}%>
globalMap.put("<%=cid %>_EXISTS", true);
} else {
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - '" + <%=directoryname %> + "' doesn't exist in the remote directory '"+ <%=remoteDir %> + "' .");
<%}%>
globalMap.put("<%=cid %>_EXISTS", false);
}
globalMap.put("<%=cid %>_DIRECTORYNAME", <%=directoryname %>);
<%}
}%>

View File

@@ -56,8 +56,6 @@ if (("true").equals(useExistingConn)) {
} else {
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
%>
<%
}
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
String passwordFieldName = "";
@@ -153,7 +151,7 @@ if (sftp) {// *** sftp *** //
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
<%if(use_encoding) {%>
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
@@ -234,104 +232,255 @@ if (sftp) {// *** sftp *** //
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%>);
nb_file_<%=cid%>++;
<%
} else {
if (ftps) { // *** ftps *** //
} else if (ftps) { // *** ftps *** //
if (("false").equals(useExistingConn)) {
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
try {
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
if (isHTTPProxyUsed) {
String httpProxyHost = System.getProperty("http.proxyHost");
int httpProxyPort = Integer.getInteger("http.proxyPort");
String httpProxyUser = System.getProperty("http.proxyUser");
String httpProxyPass = System.getProperty("http.proxyPassword");
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
} else {
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
}
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
<%
if (("false").equals(useExistingConn)) {
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
if(isLog4jEnabled){
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%
} else {
}
passwordFieldName = "__PASSWORD__";
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
if (!isLoginSuccessful_<%=cid %>) {
throw new RuntimeException("Login failed");
}
<%
}
} else { // *** ftp
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
%>
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
<%
if (("true").equals(useExistingConn)) {%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%
} else {
passwordFieldName = "__PASSWORD__";
}
if (protectionLevel != null && !protectionLevel.isEmpty()) {
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
<%
}
} //common code for ftp and ftps:
}
%>
} catch (Exception e) {
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
<%}%>
throw e;
}
<%
if ("PASSIVE".equals(connectMode)) {
%>
ftp_<%=cid %>.enterLocalPassiveMode();
<%
if (isLog4jEnabled) {
%>
log.debug("<%=cid%> - Using the passive mode.");
<%
}
}
} else {
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
String rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
<%
}
%>
int nb_file_<%=cid%> = 0;
org.apache.commons.net.ftp.FTPFile[] ftpFiles_<%=cid %> = null;
List<org.apache.commons.net.ftp.FTPFile> fileListTemp_<%=cid %> = new java.util.ArrayList<>();
<%/*read files*/ %>
String remotedir_<%=cid %> = (<%=remotedir%>).replaceAll("\\\\","/");
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
if (!cwdSuccess_<%=cid %>) {
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
}
ftpFiles_<%=cid %> = ftp_<%=cid %>.listFiles();
<%
if ("true".equals(useExistingConn) && !moveToCurrentDir) {
%>
<%/*return to previous dir */%>
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
<%
}
%>
<%/*prepare masks*/ %>
for (String filemask_<%=cid %> : maskList_<%=cid %>) {
java.util.regex.Pattern fileNamePattern_<%=cid %> = java.util.regex.Pattern.compile(filemask_<%=cid %>.replaceAll("\\.", "\\\\.").replaceAll("\\*", ".*"));
for (org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> : ftpFiles_<%=cid %>) {
if (fileNamePattern_<%=cid %>.matcher(ftpFile_<%=cid %>.getName()).matches()) {
fileListTemp_<%=cid %>.add(ftpFile_<%=cid %>);
}
}
}
String currentFilePath_<%=cid%> = remotedir_<%=cid %>;
if(!remotedir_<%=cid%>.endsWith("/")&&!remotedir_<%=cid%>.endsWith("\\")){
currentFilePath_<%=cid%> += "/";
}
for (int i = 0; i < fileListTemp_<%=cid %>.size(); i++) {
org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> = fileListTemp_<%=cid %>.get(i);
String currentFileName_<%=cid%> = null;
<%
if (dirFull) {
%>
currentFileName_<%=cid%> = ftpFile_<%=cid %>.toString();
<%
} else {
%>
currentFileName_<%=cid%> = ftpFile_<%=cid %>.getName();
<%
}
if(isLog4jEnabled) {
%>
log.debug("<%=cid%> - List file : '" + currentFilePath_<%=cid%> + "' .");
<%
}
%>
int nb_file_<%=cid%> = 0;
org.apache.commons.net.ftp.FTPFile[] ftpFiles_<%=cid %> = null;
String rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
List<org.apache.commons.net.ftp.FTPFile> fileListTemp_<%=cid %> = new java.util.ArrayList<>();
<%/*read files*/ %>
String remotedir_<%=cid %> = (<%=remotedir%>).replaceAll("\\\\","/");
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
if (!cwdSuccess_<%=cid %>) {
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
}
ftpFiles_<%=cid %> = ftp_<%=cid %>.listFiles();
<%/*set to globalMap*/ %>
globalMap.put("<%=cid %>_CURRENT_FILE", currentFileName_<%=cid%>);
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%> + ftpFile_<%=cid %>.getName());
nb_file_<%=cid %>++;
<%
if ("true".equals(useExistingConn) && !moveToCurrentDir) {
} else { // *** ftp *** //
%>
<%/*return to previous dir */%>
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
<%
}
%>
<%/*prepare masks*/ %>
for (String filemask_<%=cid %> : maskList_<%=cid %>) {
java.util.regex.Pattern fileNamePattern_<%=cid %> = java.util.regex.Pattern.compile(filemask_<%=cid %>.replaceAll("\\.", "\\\\.").replaceAll("\\*", ".*"));
int nb_file_<%=cid%> = 0;
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> =null;
for (org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> : ftpFiles_<%=cid %>) {
if (fileNamePattern_<%=cid %>.matcher(ftpFile_<%=cid %>.getName()).matches()) {
fileListTemp_<%=cid %>.add(ftpFile_<%=cid %>);
}
}
}
String currentFilePath_<%=cid%> = remotedir_<%=cid %>;
if(!remotedir_<%=cid%>.endsWith("/")&&!remotedir_<%=cid%>.endsWith("\\")){
currentFilePath_<%=cid%> += "/";
}
for (int i = 0; i < fileListTemp_<%=cid %>.size(); i++) {
org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> = fileListTemp_<%=cid %>.get(i);
String currentFileName_<%=cid%> = null;
<%if (("true").equals(useExistingConn)) {%>
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%if(!moveToCurrentDir){%>
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
<%}%>
<%} else {%>
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
ftp_<%=cid %>.setRemoteHost(<%=host %>);
ftp_<%=cid %>.setRemotePort(<%=port %>);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
<%}%>
<%if (("ACTIVE").equals(connectMode)) {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
<%} else {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
<%}%>
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
ftp_<%=cid %>.connect();
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
<%}%>
String remotedir_<%=cid %> = (<%=remotedir%>).replaceAll("\\\\","/");
ftp_<%=cid %>.chdir(remotedir_<%=cid %>);
String[] fileList_<%=cid %> = ftp_<%=cid %>.dir(null, true);
String[] fileListwithNames_<%=cid %> = ftp_<%=cid %>.dir(null, false);
<%
if (dirFull) {
if (("true").equals(useExistingConn)) {
if(!moveToCurrentDir){
%>
currentFileName_<%=cid%> = ftpFile_<%=cid %>.toString();
<%
} else {
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
<%
}
}
%>
currentFileName_<%=cid%> = ftpFile_<%=cid %>.getName();
<%
}
List<String> fileListTemp_<%=cid %> = new java.util.ArrayList<String>();
List<String> fileListTempWithNames_<%=cid %> = new java.util.ArrayList<String>();
if(isLog4jEnabled) {
%>
log.debug("<%=cid%> - List file : '" + currentFilePath_<%=cid%> + "' .");
<%
}
%>
<%/*set to globalMap*/ %>
globalMap.put("<%=cid %>_CURRENT_FILE", currentFileName_<%=cid%>);
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%> + ftpFile_<%=cid %>.getName());
nb_file_<%=cid %>++;
<%
} //end common code part
%>
for (String filemask_<%=cid %> : maskList_<%=cid %>) {
java.util.regex.Pattern fileNamePattern_<%=cid %> = java.util.regex.Pattern.compile(filemask_<%=cid %>.replaceAll("\\.", "\\\\.").replaceAll("\\*", ".*"));
for (int i_<%=cid%> = 0; i_<%=cid%> < fileList_<%=cid%>.length; i_<%=cid%>++){
String filemaskTemp_<%=cid%> = fileList_<%=cid%>[i_<%=cid%>];
String filemaskTempName_<%=cid%> = fileListwithNames_<%=cid%>[i_<%=cid%>];
if (fileNamePattern_<%=cid %>.matcher(filemaskTempName_<%=cid %>).matches()) {
fileListTemp_<%=cid %>.add(filemaskTemp_<%=cid %>);
fileListTempWithNames_<%=cid%>.add(fileListwithNames_<%=cid%>[i_<%=cid%>]);
}
}
}
int i_<%=cid %> = -1;
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Listing files from server.");
<%}%>
while (++i_<%=cid %> < fileListTemp_<%=cid %>.size()) {
<% if (dirFull){%>
String currentFileName_<%=cid%> = fileListTemp_<%=cid %>.get(i_<%=cid %>);
<% } else {%>
String currentFileName_<%=cid%> = fileListTempWithNames_<%=cid %>.get(i_<%=cid %>);
<% }%>
String currentFilePath_<%=cid%> = remotedir_<%=cid %>;
if(!remotedir_<%=cid%>.endsWith("/")&&!remotedir_<%=cid%>.endsWith("\\")){
currentFilePath_<%=cid%> += "/";
}
currentFilePath_<%=cid%> += fileListTempWithNames_<%=cid%>.get(i_<%=cid%>);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - List file : '" + currentFilePath_<%=cid%> + "' .");
<%}%>
globalMap.put("<%=cid %>_CURRENT_FILE", currentFileName_<%=cid%>);
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%>);
nb_file_<%=cid%>++;
<%}%>

View File

@@ -65,14 +65,14 @@ if (sftp && !("true").equals(useExistingConn)) {// *** sftp *** //
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
} catch (java.io.IOException ioe_<%=cid%>) {
} catch (java.net.SocketException se_<%=cid%>) {
//ignore failure
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
<%}%>
}
<%
@@ -81,7 +81,7 @@ if (sftp && !("true").equals(useExistingConn)) {// *** sftp *** //
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>

View File

@@ -155,7 +155,7 @@
<DEFAULT>"."</DEFAULT>
</PARAMETER>
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="41" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="41" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
<DEFAULT>true</DEFAULT>
</PARAMETER>
@@ -258,12 +258,12 @@
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
</IMPORTS>
</CODEGENERATION>
<RETURNS>

View File

@@ -62,8 +62,6 @@ if (("true").equals(useExistingConn)) {
} else {
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
%>
<%
}
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
@@ -155,9 +153,9 @@ if (sftp) { // *** sftp *** //
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
<%if(use_encoding) {%>
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
<%}%>
@@ -204,52 +202,93 @@ if (sftp) { // *** sftp *** //
<%}%>
}
<%
} else {
if (ftps) { // *** ftps *** //
} else if (ftps) { // *** ftps *** //
if (("false").equals(useExistingConn)) {
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
<%
if (("false").equals(useExistingConn)) {
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
<%
} else {
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
<%
}
} else { // *** ftp *** //
%>
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
<%
if (("true").equals(useExistingConn)) {
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%
} else {
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
<%
}
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
try {
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
if (isHTTPProxyUsed) {
String httpProxyHost = System.getProperty("http.proxyHost");
int httpProxyPort = Integer.getInteger("http.proxyPort");
String httpProxyUser = System.getProperty("http.proxyUser");
String httpProxyPass = System.getProperty("http.proxyPassword");
if("binary".equalsIgnoreCase(mode)){
%>
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
} else {
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
}
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
<%
} else {
passwordFieldName = "__PASSWORD__";
%>
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.ASCII_FILE_TYPE);
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
if (!isLoginSuccessful_<%=cid %>) {
throw new RuntimeException("Login failed");
}
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE); //to calculate md5 for binary files correctly
<%
}
} //common code for ftp and ftps:
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
%>
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
<%
}
if (protectionLevel != null && !protectionLevel.isEmpty()) {
%>
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
<%
}
%>
} catch (Exception e) {
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
<%}%>
throw e;
}
<%
if ("PASSIVE".equals(connectMode)) {
%>
ftp_<%=cid %>.enterLocalPassiveMode();
<%
if (isLog4jEnabled) {
%>
log.debug("<%=cid%> - Using the passive mode.");
<%
}
}
} else {
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
<%
}
%>
String remoteDir_<%=cid %> = (<%=remoteDir %>).replaceAll("\\\\", "/");
org.apache.commons.net.ftp.FTPFile[] allFiles_<%=cid %> = ftp_<%=cid %>.listFiles(remoteDir_<%=cid %>);
@@ -291,5 +330,84 @@ if (sftp) { // *** sftp *** //
<%}%>
}
<%
} else {// *** ftp *** //
%>
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> =null;
<%if (("true").equals(useExistingConn)) {%>
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%} else {%>
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
ftp_<%=cid %>.setRemoteHost(<%=host %>);
ftp_<%=cid %>.setRemotePort(<%=port %>);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
<%}%>
<%if (("ACTIVE").equals(connectMode)) {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
<%} else {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
<%}%>
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect();
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
ftp_<%=cid %>.login(<%=username %>, decryptedPassword_<%=cid%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
<%}%>
<%
//add feature 19709,add the "Transfer mode" option,
//the default transfer mode is ASCII,we don't change the mode when ascii is choosed.
if("binary".equalsIgnoreCase(mode)){
%>
ftp_<%=cid %>.setType(com.enterprisedt.net.ftp.FTPTransferType.BINARY);
<%
}
%>
String remoteDir_<%=cid %> = <%=remoteDir %>.replaceAll("\\\\", "/");
<%=outputConnName %> = new <%=outputConnName %>Struct();
String remoteFile_<%=cid %> = <%=remoteDir %> + "/" + <%=filename%>;
if (ftp_<%=cid %>.exists(remoteFile_<%=cid %>)) {
<%=outputConnName %>.abs_path = (remoteFile_<%=cid %>).replaceAll("//", "/");
<%=outputConnName %>.dirname = <%=remoteDir %>;
<%=outputConnName %>.basename = <%=filename%>;
<%=outputConnName %>.size = ftp_<%=cid %>.size(remoteFile_<%=cid %>);
<%=outputConnName %>.mtime = ftp_<%=cid %>.modtime(remoteFile_<%=cid %>).getTime();
<%=outputConnName %>.mtime_string =ftp_<%=cid %>.modtime(remoteFile_<%=cid %>).toString();
<%if(isLog4jEnabled){%>
log4jSb_<%=cid%>.append("<%=cid%> - File properties for remoteDir is : '" + <%=remoteDir %> + "',file name is : '" + <%=filename%> +"',file size is : '" + <%=outputConnName %>.size + "',file last modify time is : '" + <%=outputConnName %>.mtime_string + "'");
<%}%>
<%if (MD5) {%>
// Calculation of the Message Digest MD5
java.security.MessageDigest dgs_<%=cid %> = java.security.MessageDigest.getInstance("MD5");
dgs_<%=cid %>.update(ftp_<%=cid %>.get(remoteFile_<%=cid %>));
<%=outputConnName %>.md5 =String.format("%032x", new java.math.BigInteger(1, dgs_<%=cid %>.digest()));
<%if(isLog4jEnabled){%>
log4jSb_<%=cid%>.append(",md5 message is : '"+ <%=outputConnName %>.md5 + "'.");
<%}%>
<%}%>
<%if(isLog4jEnabled){%>
log.debug(log4jSb_<%=cid%>);
<%}%>
}
<%
}
%>

View File

@@ -80,13 +80,13 @@ if (sftp && !bUseExistingConn) { // *** sftp *** //
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
} catch (java.io.IOException ioe_<%=cid%>) {
} catch (java.net.SocketException se_<%=cid%>) {
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
<%}%>
//ignore failure
}
@@ -94,7 +94,7 @@ if (sftp && !bUseExistingConn) { // *** sftp *** //
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>

View File

@@ -255,12 +255,12 @@
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
</IMPORTS>
</CODEGENERATION>
<RETURNS />

View File

@@ -66,8 +66,6 @@ if ("true".equals(useExistingConn)) {
} else {
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
%>
<%
}
%>
int nb_file_<%=cid%> = 0;
@@ -288,7 +286,7 @@ if (sftp) { // *** sftp *** //
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
<%if(use_encoding) {%>
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
@@ -305,247 +303,477 @@ if (sftp) { // *** sftp *** //
c_<%=cid%>.cd(remotedir_<%=cid %>);
}
<%
} else {
%>
abstract class FTPGetter_<%=cid%> {
protected org.apache.commons.net.ftp.FTPClient ftpClient = null;
protected int count = 0;
} else if (!ftps) { // *** ftp *** //
%>
final com.enterprisedt.net.ftp.TransferCompleteStrings msg_<%=cid%> = new com.enterprisedt.net.ftp.TransferCompleteStrings();
public void getAllFiles(String remoteDirectory, String localDirectory)
throws IllegalStateException, IOException, java.io.FileNotFoundException {
class FTPGetter_<%=cid%> {
private com.enterprisedt.net.ftp.FTPClient ftpClient = null;
private int count = 0;
chdir(remoteDirectory);
String path = ftpClient.printWorkingDirectory();
org.apache.commons.net.ftp.FTPFile[] ftpFiles = null;
ftpFiles = ftpClient.listFiles();
public void getAllFiles(String remoteDirectory, String localDirectory) throws java.io.IOException,
com.enterprisedt.net.ftp.FTPException, java.text.ParseException {
for (org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
chdir(remoteDirectory);
String path = ftpClient.pwd();
String[] ftpFileNames = null;
com.enterprisedt.net.ftp.FTPFile[] ftpFiles = null;
if (ftpFile.isDirectory()) {
try{
//use dir() for Bug9562 with FTP server in AS400
ftpFileNames = ftpClient.dir(null, false);
//Bug 13272, the same as getFiles().
} catch (com.enterprisedt.net.ftp.FTPException e){
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - " + e.getMessage());
<%}%>
ftpFileNames = null;
ftpFiles = ftpClient.dirDetails(".");
}
if ((!(".").equals(ftpFile.getName())) && (!("..").equals(ftpFile.getName()))) {
java.io.File localFile = new java.io.File(localDirectory + "/" + ftpFile.getName());
//if dirDetails(...) doesn't work, then use dir(...), distinguish file type by FTPException
if (!localFile.exists()) {
localFile.mkdir();
}
getAllFiles(path + "/" + ftpFile.getName(), localDirectory + "/" + ftpFile.getName());
chdir(path);
}
} else if (!ftpFile.isSymbolicLink()) {
downloadFile(localDirectory + "/" + ftpFile.getName(),ftpFile.getName());
}
}
}
if ((ftpFiles == null) && (ftpFileNames != null)){
//if the file is folder, catch the FTPException and recur
for (String ftpFileName : ftpFileNames){
try {
downloadFile(localDirectory + "/" + ftpFileName,ftpFileName);
} catch (com.enterprisedt.net.ftp.FTPException e) {
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - " + e.getMessage());
<%}%>
java.io.File localFile = new java.io.File(localDirectory + "/" + ftpFileName);
public void getFiles(String remoteDirectory, String localDirectory, String maskStr)
throws IllegalStateException, IOException, java.io.FileNotFoundException {
if (!localFile.exists()) {
localFile.mkdir();
}
getAllFiles(path + "/" + ftpFileName, localDirectory + "/" + ftpFileName);
chdir(path);
}
}
} else {
for (com.enterprisedt.net.ftp.FTPFile ftpFile : ftpFiles) {
chdir(remoteDirectory);
org.apache.commons.net.ftp.FTPFile[] ftpFiles = null;
ftpFiles = ftpClient.listFiles(".");
if (ftpFile.isDir()) {
if ((!(".").equals(ftpFile.getName())) && (!("..").equals(ftpFile.getName()))) {
java.io.File localFile = new java.io.File(localDirectory + "/" + ftpFile.getName());
for(org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
if (!localFile.exists()) {
localFile.mkdir();
}
getAllFiles(path + "/" + ftpFile.getName(), localDirectory + "/" + ftpFile.getName());
chdir(path);
}
} else if (!ftpFile.isLink()) {
downloadFile(localDirectory + "/" + ftpFile.getName(),ftpFile.getName());
}
}
}
}
if (ftpFile.isFile()) {
String fileName = ftpFile.getName();
public void getFiles(String remoteDirectory, String localDirectory, String maskStr) throws java.io.IOException,
com.enterprisedt.net.ftp.FTPException, java.text.ParseException {
chdir(remoteDirectory);
String[] ftpFileNames = null;
com.enterprisedt.net.ftp.FTPFile[] ftpFiles = null;
if (fileName.matches(maskStr)) {
downloadFile(localDirectory + "/" + fileName,fileName);
}
}
}
}
public void chdir(String path) throws IllegalStateException, IOException {
if (!".".equals(path)) {
ftpClient.changeWorkingDirectory(path);
}
}
public String pwd() throws IllegalStateException, IOException {
return ftpClient.printWorkingDirectory();
}
protected abstract void downloadFile(String localFileName, String remoteFileName)
throws IllegalStateException, java.io.FileNotFoundException, IOException;
}
<%
if (!ftps) { // *** ftp *** //
%>
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
<%
if (("true").equals(useExistingConn)) {
String conn= "conn_" + connection;
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
<%if(!moveToCurrentDir){%>
String rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
<%}%>
try {
//use dir() for Bug9562 with FTP server in AS400 (the same way as getAllFiles())
ftpFileNames = ftpClient.dir(null, false);
//Bug 13272, if dir() throw exception, use dirDetails().
} catch (com.enterprisedt.net.ftp.FTPException e){
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%
} else {
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
<%
}
log.warn("<%=cid%> - " + e.getMessage());
<%}%>
ftpFileNames = null;
ftpFiles = ftpClient.dirDetails(".");
}
//if dirDetails(...) doesn't work, then use dir(...), but can not distinguish file type
if("binary".equalsIgnoreCase(ElementParameterParser.getValue(node, "__MODE__"))){
%>
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
<%
} else {
%>
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.ASCII_FILE_TYPE);
<%
}
%>
final java.util.List<String> msg_<%=cid%> = new java.util.ArrayList<String>();
FTPGetter_<%=cid%> getter_<%=cid%> = new FTPGetter_<%=cid%>() {
@Override
protected void downloadFile(String localFileName, String remoteFileName)
throws IllegalStateException, java.io.FileNotFoundException, IOException {
java.io.File localFile = new java.io.File(localFileName);
<%
if (append) {
%>
try (java.io.FileOutputStream localFos = new java.io.FileOutputStream(localFileName, true)) {
ftpClient.retrieveFile(remoteFileName, localFos);
if ((ftpFiles == null) && (ftpFileNames != null)){
for (String ftpFileName : ftpFileNames) {
if (ftpFileName.matches(maskStr)) {
downloadFile(localDirectory + "/" + ftpFileName,ftpFileName);
}
}
} else{
for (com.enterprisedt.net.ftp.FTPFile ftpFile : ftpFiles) {
if (!ftpFile.isDir() && !ftpFile.isLink()) {
String fileName = ftpFile.getName();
if (fileName.matches(maskStr)) {
downloadFile(localDirectory + "/" + fileName,fileName);
}
}
}
}
}
public void chdir(String path) throws java.io.IOException, com.enterprisedt.net.ftp.FTPException{
if (!".".equals(path)) {
ftpClient.chdir(path);
}
}
public String pwd() throws java.io.IOException, com.enterprisedt.net.ftp.FTPException{
return ftpClient.pwd();
}
private void downloadFile(String localFileName, String remoteFileName) throws java.io.IOException, com.enterprisedt.net.ftp.FTPException {
java.io.File localFile = new java.io.File(localFileName);
try {
<%if (append) {%>
java.io.FileOutputStream fout = new java.io.FileOutputStream(localFile, true);
ftpClient.get(fout, remoteFileName);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
<%}%>
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
<%if (bPrintMsg) {%>
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
<%
} else if ("never".equals(overwrite)){
%>
if (!localFile.exists()) {
ftpClient.get(localFileName, remoteFileName);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
<%}%>
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
<%if (bPrintMsg) {%>
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
} else {
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
log.info("<%= cid %> - file ["+ remoteFileName +"] exit transmission.");
<%}%>
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
<%if (bPrintMsg) {%>
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
count++;
} catch (IOException e) {
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded unsuccessfully.");
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer fail.");
throw e;
}
<%
} else if ("never".equals(overwrite)) {
%>
if (!localFile.exists()) {
downloadFileWithOverwrite(localFileName, remoteFileName);
} else {
<%if(isLog4jEnabled){%>
log.info("<%= cid %> - file ["+ remoteFileName +"] exit transmission.");
<%}%>
msg_<%=cid%>.add("file ["+ remoteFileName +"] exit transmission.");
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file transfered.");
}
<%
} else if ("always".equals(overwrite)){
%>
downloadFileWithOverwrite(localFileName, remoteFileName);
<%
} else if ("size_differ".equals(overwrite)) {
%>
if (localFile.exists()) {
long ftpSize = java.util.Arrays.stream(ftpClient.listFiles(remoteFileName)).findFirst().get().getSize();
long localSize = localFile.length();
if (ftpSize != localSize) {
downloadFileWithOverwrite(localFileName, remoteFileName);
} else {
<%if(isLog4jEnabled){%>
msg_<%=cid%>.add("file ["+ remoteFileName +"] exit transmission.");
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file transfered.");
}
<%
} else if ("always".equals(overwrite)){
%>
ftpClient.get(localFileName, remoteFileName);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
<%}%>
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
<%if (bPrintMsg) {%>
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
<%}%>
<%
} else if ("size_differ".equals(overwrite)) {
%>
if (localFile.exists()) {
long ftpSize = ftpClient.size(remoteFileName);
long localSize = localFile.length();
if (ftpSize != localSize) {
ftpClient.get(localFileName, remoteFileName);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
<%}%>
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
<%if (bPrintMsg) {%>
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
} else {
<%if(isLog4jEnabled){%>
log.info("<%= cid %> - file ["+ remoteFileName +"] exit transmission.");
<%}%>
msg_<%=cid%>.add("file ["+ remoteFileName +"] exit transmission.");
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file transfered.");
}
} else {
downloadFileWithOverwrite(localFileName, remoteFileName);
}
<%
}
%>
}
private void downloadFileWithOverwrite(String localFileName, String remoteFileName)
throws IllegalStateException, java.io.FileNotFoundException, IOException {
try (java.io.FileOutputStream localFos = new java.io.FileOutputStream(localFileName)) {
ftpClient.retrieveFile(remoteFileName, localFos);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
<%}%>
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
<%if (bPrintMsg) {%>
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
count++;
} catch (IOException e) {
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded unsuccessfully.");
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer fail.");
throw e;
}
}
};
getter_<%=cid%>.ftpClient = ftp_<%=cid %>;
<%
} else { // *** ftps *** //
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
<%
if ("false".equals(useExistingConn)){
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
<%
} else {
String conn= "conn_" + connection;
%>
<% /* get connection from globalMap*/ %>
String rootDir_<%=cid %> = null;
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient)globalMap.get("<%=conn %>");
<%
if (!moveToCurrentDir) {
%>
rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
<%
}
if(isLog4jEnabled) {
%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection.Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%
}
}
%>
FTPGetter_<%=cid%> getter_<%=cid%> = new FTPGetter_<%=cid%>() {
@Override
protected void downloadFile(String localFileName, String remoteFileName)
throws IllegalStateException, java.io.FileNotFoundException, IOException {
try (java.io.FileOutputStream localFos = new java.io.FileOutputStream(localFileName)) {
ftpClient.retrieveFile(remoteFileName, localFos);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
<%}%>
count++;
}
}
};
getter_<%=cid%>.ftpClient = ftp_<%=cid %>;
<%
} //common code for ftp and ftps:
%>
String remotedir_<%=cid%> = <%=remotedir%>;
if (!".".equals(remotedir_<%=cid %>)) {
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
if (!cwdSuccess_<%=cid %>) {
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
}
msg_<%=cid%>.add("file ["+ remoteFileName +"] exit transmission.");
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file transfered.");
}
} else {
ftpClient.get(localFileName, remoteFileName);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
<%}%>
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
<%if (bPrintMsg) {%>
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
}
<%
}
%>
} catch (com.enterprisedt.net.ftp.FTPException e) {
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded unsuccessfully.");
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer fail.");
throw e;
}
count++;
}
}
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> = null;
<%
if ("true".equals(useExistingConn)){
String conn= "conn_" + connection;
%>
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
<%if(!moveToCurrentDir){%>
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
<%}%>
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Uses an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%} else {%>
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
ftp_<%=cid %>.setRemoteHost(<%=host %>);
ftp_<%=cid %>.setRemotePort(<%=port %>);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
<%}
if (("ACTIVE").equals(connectMode)){
%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
<%
}else{
%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
<%
}
%>
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect();
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
<%} %>
msg_<%=cid%>.clearAll();
FTPGetter_<%=cid%> getter_<%=cid%> = new FTPGetter_<%=cid%>();
getter_<%=cid%>.ftpClient = ftp_<%=cid %>;
String remotedir_<%=cid %> = <%=remotedir%>;
if (!".".equals(remotedir_<%=cid %>)) {
ftp_<%=cid %>.chdir(remotedir_<%=cid %>);
}
<%
} else { // *** ftps *** //
%>
class FTPSGetter_<%=cid%> {
private org.apache.commons.net.ftp.FTPClient ftpClient = null;
private int count = 0;
public void getAllFiles(String remoteDirectory, String localDirectory)
throws IllegalStateException, IOException, java.io.FileNotFoundException {
chdir(remoteDirectory);
String path = ftpClient.printWorkingDirectory();
org.apache.commons.net.ftp.FTPFile[] ftpFiles = null;
ftpFiles = ftpClient.listFiles();
for (org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
if (ftpFile.isDirectory()) {
if ((!(".").equals(ftpFile.getName())) && (!("..").equals(ftpFile.getName()))) {
java.io.File localFile = new java.io.File(localDirectory + "/" + ftpFile.getName());
if (!localFile.exists()) {
localFile.mkdir();
}
getAllFiles(path + "/" + ftpFile.getName(), localDirectory + "/" + ftpFile.getName());
chdir(path);
}
} else if (!ftpFile.isSymbolicLink()) {
downloadFile(localDirectory + "/" + ftpFile.getName(),ftpFile.getName());
}
}
}
public void getFiles(String remoteDirectory, String localDirectory, String maskStr)
throws IllegalStateException, IOException, java.io.FileNotFoundException {
chdir(remoteDirectory);
org.apache.commons.net.ftp.FTPFile[] ftpFiles = null;
ftpFiles = ftpClient.listFiles(".");
for(org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
if (ftpFile.isFile()) {
String fileName = ftpFile.getName();
if (fileName.matches(maskStr)) {
downloadFile(localDirectory + "/" + fileName,fileName);
}
}
}
}
public void chdir(String path)
throws IllegalStateException, IOException {
if (!".".equals(path)) {
ftpClient.changeWorkingDirectory(path);
}
}
public String pwd()
throws IllegalStateException, IOException {
return ftpClient.printWorkingDirectory();
}
private void downloadFile(String localFileName, String remoteFileName)
throws IllegalStateException, java.io.FileNotFoundException, IOException {
try (java.io.FileOutputStream localFos = new java.io.FileOutputStream(localFileName)) {
ftpClient.retrieveFile(remoteFileName, localFos);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
<%}%>
count++;
}
}
}
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
<%
if ("false".equals(useExistingConn)){
%>
<% /* create connection */ %>
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
javax.net.ssl.SSLContext sslContext_<%=cid%> = null;
javax.net.ssl.TrustManager[] trustManager_<%=cid%> = null;
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid%> = null;
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
try {
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
sslSocketFactory_<%=cid %> = sslContext_<%=cid%>.getSocketFactory();
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
if (isHTTPProxyUsed) {
String httpProxyHost = System.getProperty("http.proxyHost");
int httpProxyPort = Integer.getInteger("http.proxyPort");
String httpProxyUser = System.getProperty("http.proxyUser");
String httpProxyPass = System.getProperty("http.proxyPassword");
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
} else {
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
}
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
if (!isLoginSuccessful_<%=cid %>) {
throw new RuntimeException("Login failed");
}
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
<%
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
%>
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
<%
}
if (protectionLevel != null && !protectionLevel.isEmpty()) {
%>
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
<%
}
%>
} catch (Exception e) {
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
<%}%>
<%
if (dieOnError) {
%>
throw e;
<%
} else {
%>
e.printStackTrace();
<%
}
%>
}
<%
if ("PASSIVE".equals(connectMode)) {
%>
ftp_<%=cid %>.enterLocalPassiveMode();
<%
if (isLog4jEnabled) {
%>
log.debug("<%=cid%> - Using the passive mode.");
<%
}
}
} else {
String conn= "conn_" + connection;
%>
<% /* get connection from globalMap*/ %>
String rootDir_<%=cid %> = null;
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient)globalMap.get("<%=conn %>");
<%
if (!moveToCurrentDir) {
%>
rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
<%
}
if(isLog4jEnabled) {
%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection.Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%
}
}
%>
FTPSGetter_<%=cid%> getter_<%=cid%> = null;
String remotedir_<%=cid%> = null;
getter_<%=cid%> = new FTPSGetter_<%=cid%>();
getter_<%=cid%>.ftpClient = ftp_<%=cid %>;
remotedir_<%=cid%> = <%=remotedir%>;
if (!".".equals(remotedir_<%=cid %>)) {
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
if (!cwdSuccess_<%=cid %>) {
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
}
}
<%
}
// *** share code *** //
@@ -555,10 +783,22 @@ java.util.List<String> maskList_<%=cid %> = new java.util.ArrayList<String>();
<%
for (int i = 0; i < files.size(); i++) {
Map<String, String> line = files.get(i);
%>
maskList_<%=cid %>.add(<%= line.get("FILEMASK") %>);
%>
maskList_<%=cid %>.add(<%= line.get("FILEMASK") %>);
<%
}
if (!sftp && !ftps) { // *** ftp *** //
if ("binary".equalsIgnoreCase(ElementParameterParser.getValue(node, "__MODE__"))) {
%>
ftp_<%=cid %>.setType(com.enterprisedt.net.ftp.FTPTransferType.BINARY);
<%
}else {
%>
ftp_<%=cid %>.setType(com.enterprisedt.net.ftp.FTPTransferType.ASCII);
<%
}
}
%>
String localdir_<%=cid %> = <%=localdir%>;
//create folder if local direcotry (assigned by property) not exists

View File

@@ -54,8 +54,8 @@ nb_file_<%=cid%> = getter_<%=cid %>.count;
<%}%>
<%}%>
<%} else if (!ftps) {%>
msg_<%=cid%>.add(getter_<%=cid%>.count + " files have been downloaded.");
String[] msgAll_<%=cid %> = msg_<%=cid%>.toArray(new String[0]);
msg_<%=cid%>.add(ftp_<%=cid%>.getDownloadCount() + " files have been downloaded.");
String[] msgAll_<%=cid %> = msg_<%=cid%>.getAll();
StringBuffer sb_<%=cid %> = new StringBuffer();
if (msgAll_<%=cid %> != null) {
@@ -74,13 +74,13 @@ nb_file_<%=cid%> = getter_<%=cid %>.count;
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
} catch(java.io.IOException ioe_<%=cid%>) {
} catch(java.net.SocketException se_<%=cid%>) {
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - " + ioe_<%=cid%>.getMessage());
log.error("<%=cid%> - " + se_<%=cid%>.getMessage());
<%}%>
//ignore failure
}
@@ -90,7 +90,7 @@ nb_file_<%=cid%> = getter_<%=cid %>.count;
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
@@ -99,7 +99,7 @@ nb_file_<%=cid%> = getter_<%=cid %>.count;
}else{
if(!moveToCurrentDir){
%>
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
<%
}
}

View File

@@ -150,7 +150,7 @@ COMPATIBILITY="ALL"
<DEFAULT>"."</DEFAULT>
</PARAMETER>
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="61" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="61" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
<DEFAULT>true</DEFAULT>
</PARAMETER>
@@ -298,15 +298,15 @@ COMPATIBILITY="ALL"
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="jakarta-oro" MODULE="jakarta-oro-2.0.8.jar" MVN="mvn:org.talend.libraries/jakarta-oro-2.0.8/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache/lib/jakarta-oro-2.0.8.jar" REQUIRED="true" />
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
</IMPORTS>
</CODEGENERATION>
</CODEGENERATION>
<RETURNS>
<RETURN NAME="NB_FILE" TYPE="id_Integer" AVAILABILITY="AFTER"/>
<RETURN NAME="CURRENT_STATUS" TYPE="id_String" AVAILABILITY="FLOW"/>

View File

@@ -16,10 +16,23 @@ String dieOnError = ElementParameterParser.getValue(node, "__DIE_ON_ERROR__");
String remotedir = ElementParameterParser.getValue(node, "__REMOTEDIR__");
String connection = ElementParameterParser.getValue(node, "__CONNECTION__");
String useExistingConn = ElementParameterParser.getValue(node, "__USE_EXISTING_CONNECTION__");
boolean sftp = false;
boolean usePerl5 = ("true").equals(ElementParameterParser.getValue(node, "__PERL5_REGEX__"));
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
if ("true".equals(useExistingConn)) {
List<? extends INode> nodeList = node.getProcess().getGeneratingNodes();
for (INode n : nodeList) {
if (n.getUniqueName().equals(connection)) {
sftp = "true".equals(ElementParameterParser.getValue(n, "__SFTP__"));
}
}
} else {
sftp = "true".equals(ElementParameterParser.getValue(node, "__SFTP__"));
}
%>
try {

View File

@@ -62,10 +62,8 @@ if (("true").equals(useExistingConn)) {
}
} else {
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
%>
<%
}
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
}
//The following part support the socks proxy for FTP and SFTP (Socks V4 or V5, they are all OK).
//And it can not work with the FTP proxy directly, only support the socks proxy.
@@ -152,8 +150,8 @@ int nb_file_<%=cid%> = 0;
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
<%if(use_encoding) {%>
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
@@ -230,103 +228,255 @@ int nb_file_<%=cid%> = 0;
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
<%} else {
if (!ftps) {%> // *** ftp *** //
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
<%
if (("true").equals(useExistingConn)) {
String conn= "conn_" + connection;
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%
} else {
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
<%
}
<%} else if (!ftps) {%> // *** ftp *** //
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> =null;
if("binary".equalsIgnoreCase(ElementParameterParser.getValue(node, "__MODE__"))){
%>
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
<%
} else {
%>
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.ASCII_FILE_TYPE);
<%
}
%>
// msg_<%=cid%> likes a String[] to save the message from transfer.
java.util.List<String> msg_<%=cid%> = new java.util.ArrayList<>();
<%
} else { // *** ftps *** //
String keystoreFile = ElementParameterParser.getValue(node,"__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
<%
if(("false").equals(useExistingConn)){
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
<%
} else {
String conn= "conn_" + connection;
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient)globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Uses an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%
}
} //common code for ftp and ftps:
%>
String rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
String remotedir_<%=cid%> = (<%=remotedir%>).replaceAll("\\\\", "/");
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
<%
if(createDirIfNotExist) {
%>
if(!cwdSuccess_<%=cid %>){
String[] dirsTree_<%=cid %> = remotedir_<%=cid %>.split("/");
for (String dir: dirsTree_<%=cid %>) {
ftp_<%=cid %>.makeDirectory(dir);
ftp_<%=cid %>.changeWorkingDirectory(dir);
<%
if (("true").equals(useExistingConn)) {
String conn= "conn_" + connection;
%>
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Uses an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
<%}%>
<%} else {%>
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
ftp_<%=cid %>.setRemoteHost(<%=host %>);
ftp_<%=cid %>.setRemotePort(<%=port %>);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
<%}%>
<%if (("ACTIVE").equals(connectMode)) {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
<%} else {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
<%}%>
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect();
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
<%}%>
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
// msg_<%=cid%> likes a String[] to save the message from transfer.
com.enterprisedt.net.ftp.TransferCompleteStrings msg_<%=cid%> = ftp_<%=cid %>.getTransferCompleteMessages();
msg_<%=cid%>.clearAll();
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
<%
for (int i = 0; i < files.size(); i++) {
Map<String, String> line = files.get(i);
%>
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,<%= line.get("NEWNAME") %>);
list<%=cid %>.add(map<%=cid %><%=i %>);
<%
}
%>
String remotedir<%=cid %> = (<%=remotedir%>).replaceAll("\\\\", "/");
<%
if (createDirIfNotExist) {
%>
boolean isDirectoryExist_<%=cid %> = ftp_<%=cid %>.existsDirectory(remotedir<%=cid %>);
if (!isDirectoryExist_<%=cid %>) {
String rootDir = null;
String[] dirsTree_<%=cid %> = remotedir<%=cid %>.split("/");
for (String dir: dirsTree_<%=cid %>) {
if (!ftp_<%=cid %>.existsDirectory(dir)) {
ftp_<%=cid %>.mkdir(dir);
}
ftp_<%=cid %>.chdir(dir);
}
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
}
<%
}
}
%>
ftp_<%=cid %>.chdir(remotedir<%=cid %>);
<%if ("binary".equalsIgnoreCase(ElementParameterParser.getValue(node, "__MODE__"))) {%>
ftp_<%=cid %>.setType(com.enterprisedt.net.ftp.FTPTransferType.BINARY);
<%} else { %>
ftp_<%=cid %>.setType(com.enterprisedt.net.ftp.FTPTransferType.ASCII);
<%}%>
String localdir<%=cid %> = <%=localdir%>;
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Uploading files to the server.");
<%}%>
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
<%
} else { // *** ftps *** //
String keystoreFile = ElementParameterParser.getValue(node,"__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
String rootDir_<%=cid %> = null;
<%
if(("false").equals(useExistingConn)){
%>
<% /* create connection */ %>
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
javax.net.ssl.SSLContext sslContext_<%=cid%> = null;
javax.net.ssl.TrustManager[] trustManager_<%=cid%> = null;
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid%> = null;
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
try {
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
sslSocketFactory_<%=cid %> = sslContext_<%=cid%>.getSocketFactory();
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
if (isHTTPProxyUsed) {
String httpProxyHost = System.getProperty("http.proxyHost");
int httpProxyPort = Integer.getInteger("http.proxyPort");
String httpProxyUser = System.getProperty("http.proxyUser");
String httpProxyPass = System.getProperty("http.proxyPassword");
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
} else {
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
}
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
<%
passwordFieldName = "__PASSWORD__";
%>
if (!cwdSuccess_<%=cid %>) {
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
}
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
if (!isLoginSuccessful_<%=cid %>) {
throw new RuntimeException("Login failed");
}
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
<%
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
%>
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
<%
}
if (protectionLevel != null && !protectionLevel.isEmpty()) {
%>
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
<%
}
%>
} catch (Exception e) {
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
<%}%>
<%
if (dieOnError) {
%>
throw e;
<%
} else {
%>
e.printStackTrace();
<%
}
%>
}
<%
if ("PASSIVE".equals(connectMode)) {
%>
ftp_<%=cid %>.enterLocalPassiveMode();
<%
if (isLog4jEnabled) {
%>
log.debug("<%=cid%> - Using the passive mode.");
<%
}
}
} else {
String conn= "conn_" + connection;
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient)globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Uses an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
<%
}
%>
String remotedir_<%=cid%> = (<%=remotedir%>).replaceAll("\\\\", "/");
<%
for (int i = 0; i < files.size(); i++) {
Map<String, String> line = files.get(i);
if(createDirIfNotExist) {
%>
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>, <%= line.get("NEWNAME") %>);
list<%=cid %>.add(map<%=cid %><%=i %>);
String[] dirsTree_<%=cid %> = remotedir_<%=cid %>.split("/");
for (String dir: dirsTree_<%=cid %>) {
ftp_<%=cid %>.makeDirectory(dir);
ftp_<%=cid %>.changeWorkingDirectory(dir);
}
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
<%
}
}
%>
String localdir<%=cid %> = <%=localdir%>;
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Uploading files to the server.");
<%}%>
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
if (!cwdSuccess_<%=cid %>) {
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
}
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
<%
}
for (int i = 0; i < files.size(); i++) {
Map<String, String> line = files.get(i);
%>
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>, <%= line.get("NEWNAME") %>);
list<%=cid %>.add(map<%=cid %><%=i %>);
<%
}
%>
String localdir<%=cid %> = <%=localdir%>;
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Uploading files to the server.");
<%}%>
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
<%
} /* close ftps */
%>

View File

@@ -57,9 +57,9 @@ if(sftp){ // *** sftp *** //
<%}else if(!ftps){%> // *** ftp *** //
}
msg_<%=cid%>.add(nb_file_<%=cid%> + " files have been uploaded.");
msg_<%=cid%>.add(ftp_<%=cid %>.getUploadCount() + " files have been uploaded.");
String[] msgAll_<%=cid %> = msg_<%=cid%>.toArray(new String[0]);
String[] msgAll_<%=cid %> = msg_<%=cid%>.getAll();
StringBuffer sb_<%=cid %> = new StringBuffer();
if (msgAll_<%=cid %> != null) {
for (String item_<%=cid %> : msgAll_<%=cid %>) {
@@ -76,13 +76,13 @@ if(sftp){ // *** sftp *** //
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
}catch(java.io.IOException ioe_<%=cid%>){
}catch(java.net.SocketException se_<%=cid%>){
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
<%}%>
//ignore failure
}
@@ -92,7 +92,7 @@ if(sftp){ // *** sftp *** //
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
@@ -101,7 +101,7 @@ if(sftp){ // *** sftp *** //
}else{
if(!moveToCurrentDir){
%>
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
<%
}
}

View File

@@ -165,7 +165,7 @@ COMPATIBILITY="ALL"
<DEFAULT>false</DEFAULT>
</PARAMETER>
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="6" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="6" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
<DEFAULT>true</DEFAULT>
</PARAMETER>
@@ -343,12 +343,12 @@ COMPATIBILITY="ALL"
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
</IMPORTS>
</CODEGENERATION>
<RETURNS>

View File

@@ -185,14 +185,125 @@ try{
}
}
<%
} else {// *** ftp+ ftps *** //
if (!ftps) {
ftpsoverwrite = overwrite; //to avoid migration, different dropdowns with same purpose
}else if (!ftps) { // *** ftp *** //
%>
String currentStatus_<%=cid %> = "No file transfered.";
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
for (String key<%=cid %> : keySet<%=cid %>){
if(key<%=cid %> == null || "".equals(key<%=cid%>)){
System.err.println("file name invalid!");
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - file name invalid!");
<%}%>
continue;
}
String tempdir<%=cid %> = localdir<%=cid %>;
String filemask<%=cid %> = key<%=cid %>;
String dir<%=cid %> = null;
<%if(!useGlob) {//perl5 mode not support windows(\) path separator at the mask string%>
String mask<%=cid %> = filemask<%=cid %>.replaceAll("\\\\", "/") ;
<%} else {%>
String mask<%=cid %> = filemask<%=cid%>;
<%}%>
int i<%=cid %> = mask<%=cid %>.lastIndexOf('/');
if (i<%=cid %>!=-1){
dir<%=cid %> = mask<%=cid %>.substring(0, i<%=cid %>);
mask<%=cid %> = mask<%=cid %>.substring(i<%=cid %>+1);
}
if (dir<%=cid %>!=null && !"".equals(dir<%=cid %>)){
tempdir<%=cid %> = tempdir<%=cid %> + "/" + dir<%=cid %>;
}
<%if(!useGlob) {%>
mask<%=cid %> = mask<%=cid %>.replaceAll("\\.", "\\\\.").replaceAll("\\*", ".*");
<%}%>
final String finalMask<%=cid %> = mask<%=cid %>;
java.io.File[] listings<%=cid %> = null;
java.io.File file<%=cid %> = new java.io.File(tempdir<%=cid %>);
if (file<%=cid %>.isDirectory()) {
listings<%=cid %> = file<%=cid %>.listFiles(new java.io.FileFilter() {
public boolean accept(java.io.File pathname) {
boolean result = false;
if (pathname != null && pathname.isFile()) {
result = java.util.regex.Pattern.compile(finalMask<%=cid %>).matcher(pathname.getName()).find();
}
return result;
}
});
}
if(listings<%=cid %> != null && listings<%=cid %>.length > 0){
for (int m<%=cid %> = 0; m<%=cid %> < listings<%=cid %>.length; m<%=cid %>++){
java.io.File file_in_localDir = listings<%=cid %>[m<%=cid %>];
String uploadingFileName_<%= cid %> = file_in_localDir.getName();
String newFileName_<%= cid %> = map<%=cid %>.get(key<%=cid %>);
String newName_<%=cid%> = ("".equals(newFileName_<%= cid %>)) ? uploadingFileName_<%= cid %> : newFileName_<%= cid %>;
boolean targetFileExistsOnRemoteServer_<%= cid %> = false;
try{
if (uploadingFileName_<%= cid %>.matches(mask<%=cid %>)){
<%
if(("never".equals(overwrite) || "size_differ".equals(overwrite)) && !append){
%>
targetFileExistsOnRemoteServer_<%= cid %> = ftp_<%= cid %>.existsFile(newName_<%= cid %>);
globalMap.put("<%=cid %>_CURRENT_FILE_EXISTS", targetFileExistsOnRemoteServer_<%= cid %>);
<%
}
%>
<%if(append){%>
ftp_<%=cid %>.put(tempdir<%=cid %> + "/" + uploadingFileName_<%= cid %>, newName_<%= cid %>, true);
<%}else if(("never").equals(overwrite)){%>
if(!targetFileExistsOnRemoteServer_<%= cid %>){
ftp_<%=cid %>.put(tempdir<%=cid %> + "/" + uploadingFileName_<%= cid %>, newName_<%=cid%>);
}
<%}else if(("always").equals(overwrite)){%>
ftp_<%=cid %>.put(tempdir<%=cid %> + "/" + uploadingFileName_<%= cid %>, newName_<%=cid%>);
<%}else if(("size_differ").equals(overwrite)){%>
if (targetFileExistsOnRemoteServer_<%= cid %>){
com.enterprisedt.net.ftp.FTPFile ftpfile<%=cid%>=ftp_<%=cid%>.fileDetails(newName_<%=cid%>);
long ftpSize<%=cid%>=ftpfile<%=cid%>.size();
long localSize<%=cid%>=file_in_localDir.length();
if(ftpSize<%=cid%> != localSize<%=cid%>){
ftp_<%=cid %>.put(tempdir<%=cid %> + "/" + uploadingFileName_<%= cid %>, newName_<%=cid%>);
}
}else{
ftp_<%=cid %>.put(tempdir<%=cid %> + "/" + uploadingFileName_<%= cid %>, newName_<%=cid%>);
}
<%}%>
currentStatus_<%=cid %> = "File rename OK.";
msg_<%=cid%>.add("file: " + file_in_localDir.getAbsolutePath() + ", size: "
+ file_in_localDir.length() + " bytes upload successfully");
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Uploaded file '" + uploadingFileName_<%= cid %> + "' successfully.");
<%}%>
nb_file_<%=cid%>++;
}
}catch(com.enterprisedt.net.ftp.FTPException e_<%=cid%>){
msg_<%=cid%>.add("file " + file_in_localDir.getAbsolutePath() + " not found?");
currentStatus_<%=cid %> = "File rename fail.";
globalMap.put("<%=cid %>_CURRENT_FILE_EXISTS", targetFileExistsOnRemoteServer_<%= cid %>);
throw e_<%=cid%>;
}
}
}else{
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - No match file exist!");
<%}%>
System.err.println("No match file exist!");
}
}
globalMap.put("<%=cid %>_CURRENT_STATUS", currentStatus_<%=cid %>);
<%
} else {// *** ftps *** //
%>
String currentStatus_<%=cid %> = "No file transfered.";
globalMap.put("<%=cid %>_CURRENT_STATUS", currentStatus_<%=cid %>);
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file transfered.");
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
boolean needRename_<%=cid%> = false;
for (String key<%=cid %> : keySet<%=cid %>){
@@ -257,43 +368,17 @@ try{
java.io.FileInputStream file_stream_<%=cid %> = new java.io.FileInputStream(file_in_localDir_<%=cid%>);
globalMap.put("<%=cid %>_CURRENT_FILE_EXISTS", remoteExistsFiles_<%=cid%>.contains(file_in_localDir_<%=cid%>.getName()));
String newName_<%=cid%> = ("".equals(map<%=cid %>.get(key<%=cid %>)))?file_in_localDir_<%=cid%>.getName():map<%=cid %>.get(key<%=cid %>);
needRename_<%=cid%> = true;
<%
if (!ftps && append) {
%>
if ((remoteExistsFiles_<%=cid%>.contains(newName_<%=cid%>))){
ftp_<%=cid %>.appendFile(file_in_localDir_<%=cid%>.getName(), file_stream_<%=cid %>);
} else {
ftp_<%=cid %>.storeFile(file_in_localDir_<%=cid%>.getName(), file_stream_<%=cid %>);
}
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Uploaded file '" + newName_<%=cid%> + "' successfully.");
<%}%>
globalMap.put("<%=cid %>_CURRENT_FILE_EXISTS", remoteExistsFiles_<%=cid%>.contains(newName_<%=cid%>));
<%
} else if ("size_differ".equals(ftpsoverwrite)) {
%>
if ((remoteExistsFiles_<%=cid%>.contains(newName_<%=cid%>))){
long ftpSize = java.util.Arrays.stream(ftp_<%=cid %>.listFiles(newName_<%=cid%>)).filter(org.apache.commons.net.ftp.FTPFile::isFile).findFirst().get().getSize();
long localSize = file_in_localDir_<%=cid%>.length();
if (ftpSize != localSize) {
ftp_<%=cid %>.deleteFile(newName_<%=cid%>);
}
}
ftp_<%=cid %>.storeFile(file_in_localDir_<%=cid%>.getName(), file_stream_<%=cid %>);
<%
} else if("never".equals(ftpsoverwrite)){
%>
<%if("never".equals(ftpsoverwrite)){%>
if (!(remoteExistsFiles_<%=cid%>.contains(newName_<%=cid%>))){
needRename_<%=cid%> = true;
ftp_<%=cid %>.storeFile(file_in_localDir_<%=cid%>.getName(), file_stream_<%=cid %>);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - Uploaded file '" + newName_<%=cid%> + "' successfully.");
<%}%>
globalMap.put("<%=cid %>_CURRENT_FILE_EXISTS", remoteExistsFiles_<%=cid%>.contains(newName_<%=cid%>));
} else {
needRename_<%=cid %> = false;
}
<%}else if("always".equals(ftpsoverwrite)){%>
needRename_<%=cid%> = true;
if ((remoteExistsFiles_<%=cid%>.contains(newName_<%=cid%>))){
ftp_<%=cid %>.deleteFile(newName_<%=cid%>);
}
@@ -306,14 +391,6 @@ try{
<%}%>
file_stream_<%=cid %>.close();
<%
if (!ftps) {
%>
msg_<%=cid%>.add("file: " + file_in_localDir_<%=cid%>.getAbsolutePath() + ", size: "
+ file_in_localDir_<%=cid%>.length() + " bytes upload successfully");
<%
}
%>
nb_file_<%=cid%>++;
}
}
@@ -341,12 +418,6 @@ try{
<%}%>
}catch(java.lang.Exception e_<%=cid%>){
<%
if (!ftps) {
%>
msg_<%=cid%>.add("file not found?: " + e_<%=cid%>.getMessage());
<%
}
if(("true").equals(dieOnError)){
%>
throw(e_<%=cid%>);

View File

@@ -57,8 +57,6 @@ if (("true").equals(useExistingConn)) {
} else {
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
%>
<%
}
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
@@ -133,9 +131,9 @@ if (sftp) { // *** sftp *** //
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
<%if(use_encoding) {%>
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
<%}%>
@@ -174,42 +172,102 @@ if (sftp) { // *** sftp *** //
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
<%
} else {
if (ftps) { // *** ftps *** //
} else if (ftps) { // *** ftps *** //
if (("false").equals(useExistingConn)) {
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
try {
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
if (isHTTPProxyUsed) {
String httpProxyHost = System.getProperty("http.proxyHost");
int httpProxyPort = Integer.getInteger("http.proxyPort");
String httpProxyUser = System.getProperty("http.proxyUser");
String httpProxyPass = System.getProperty("http.proxyPassword");
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
} else {
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
}
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
<%
if (("false").equals(useExistingConn)) {
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
if (!isLoginSuccessful_<%=cid %>) {
throw new RuntimeException("Login failed");
}
<%
} else {
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
<%
}
} else { // *** ftp *** //
}
if (protectionLevel != null && !protectionLevel.isEmpty()) {
%>
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
<%
if (("true").equals(useExistingConn)) {
}
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
} catch (Exception e) {
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
<%}%>
<%
} else {
passwordFieldName = "__PASSWORD__";
if (dieOnError) {
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
throw e;
<%
}
} //common code for ftp and ftps:
} else {
%>
e.printStackTrace();
<%
}
%>
}
<%
if ("PASSIVE".equals(connectMode)) {
%>
ftp_<%=cid %>.enterLocalPassiveMode();
<%
if (isLog4jEnabled) {
%>
log.debug("<%=cid%> - Using the passive mode.");
<%
}
}
} else {
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
<%
}
%>
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
@@ -236,5 +294,67 @@ if (sftp) { // *** sftp *** //
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) { <% /* start main loop */ %>
<%
}
} else {// *** ftp *** //
%>
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> =null;
<%if (("true").equals(useExistingConn)) {%>
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%if(!moveToCurrentDir){%>
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
<%}%>
<%} else {%>
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
ftp_<%=cid %>.setRemoteHost(<%=host %>);
ftp_<%=cid %>.setRemotePort(<%=port %>);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
<%}%>
<%if (("ACTIVE").equals(connectMode)) {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
<%} else {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
<%}%>
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect();
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
<%}%>
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
<%
for (int i = 0; i < files.size(); i++) {
Map<String, String> line = files.get(i);
%>
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,<%= line.get("NEWNAME") %>);
list<%=cid %>.add(map<%=cid %><%=i %>);
<%
}
%>
String remotedir<%=cid %> = <%=remotedir%>;
ftp_<%=cid %>.chdir(remotedir<%=cid %>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Renaming file from server.");
<%}%>
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
<%}%>

View File

@@ -78,13 +78,13 @@
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
}catch(java.io.IOException ioe_<%=cid%>){
}catch(java.net.SocketException se_<%=cid%>){
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
<%}%>
//ignore failure
}
@@ -94,7 +94,7 @@
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
@@ -103,7 +103,7 @@
}else{
if(!moveToCurrentDir){
%>
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
<%
}
}

View File

@@ -143,7 +143,7 @@ COMPATIBILITY="ALL"
<DEFAULT>"."</DEFAULT>
</PARAMETER>
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="31" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="31" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
<DEFAULT>true</DEFAULT>
</PARAMETER>
@@ -261,12 +261,12 @@ COMPATIBILITY="ALL"
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
</IMPORTS>
</CODEGENERATION>
<RETURNS>

View File

@@ -57,74 +57,25 @@ try{
}
<%
} else { // *** ftp+s *** //
} else if (ftps) { // *** ftps *** //
%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file renamed.");
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
for (String key<%=cid %> : keySet<%=cid %>) {
try{
<%
if (!ftps && "never".equals(overwrite)) {
%>
String[] remoteFileArray_<%=cid%> = java.util.Arrays.stream(ftp_<%=cid %>.listFiles()).filter(org.apache.commons.net.ftp.FTPFile::isFile).map(org.apache.commons.net.ftp.FTPFile::getName).filter(map<%=cid %>.get(key<%=cid %>)::equals).toArray(String[]::new);
boolean isRemoteFileExists_<%=cid %> = (remoteFileArray_<%=cid%> != null) && (remoteFileArray_<%=cid%>.length > 0);
if (!isRemoteFileExists_<%=cid%>) {
boolean renameSuccess_<%=cid %> = ftp_<%=cid%>.rename(key<%=cid %>, map<%=cid %>.get(key<%=cid %>));
if (renameSuccess_<%=cid %>) {
nb_file_<%=cid%>++;
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - '" + key<%=cid %> + "' rename ok.");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
} else {
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - '" + key<%=cid %> + "' rename failed. " + ftp_<%=cid %>.getReplyString());
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename fail.");
}
}
<%
} else if (!ftps && "size_differ".equals(overwrite)) {
%>
org.apache.commons.net.ftp.FTPFile toRenameFile_<%=cid %> = java.util.Arrays.stream(ftp_<%=cid %>.listFiles(key<%=cid %>)).filter(org.apache.commons.net.ftp.FTPFile::isFile).findFirst().orElse(null);
if (toRenameFile_<%=cid %> != null) {
org.apache.commons.net.ftp.FTPFile toOverwriteFile_<%=cid %> = java.util.Arrays.stream(ftp_<%=cid %>.listFiles(map<%=cid %>.get(key<%=cid %>))).filter(org.apache.commons.net.ftp.FTPFile::isFile).findFirst().orElse(null);
if ((toOverwriteFile_<%=cid %> != null) && toRenameFile_<%=cid %>.getSize() != toOverwriteFile_<%=cid%>.getSize()) {
ftp_<%=cid %>.deleteFile(map<%=cid %>.get(key<%=cid %>));
}
boolean renameSuccess_<%=cid %> = ftp_<%=cid%>.rename(key<%=cid %>, map<%=cid %>.get(key<%=cid %>));
if (renameSuccess_<%=cid %>) {
nb_file_<%=cid%>++;
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - '" + key<%=cid %> + "' rename ok.");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
} else {
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - '" + key<%=cid %> + "' rename failed. " + ftp_<%=cid %>.getReplyString());
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename fail.");
}
}
<%
} else {
%>
boolean renameSuccess_<%=cid %> = ftp_<%=cid%>.rename(key<%=cid %>, map<%=cid %>.get(key<%=cid %>));
if (renameSuccess_<%=cid %>) {
nb_file_<%=cid%>++;
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - '" + key<%=cid %> + "' rename ok.");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
} else {
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - '" + key<%=cid %> + "' rename failed. " + ftp_<%=cid %>.getReplyString());
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename fail.");
}
<%
}
%>
boolean renameSuccess_<%=cid %> = ftp_<%=cid%>.rename(key<%=cid %>, map<%=cid %>.get(key<%=cid %>));
if (renameSuccess_<%=cid %>) {
nb_file_<%=cid%>++;
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - '" + key<%=cid %> + "' rename ok.");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
} else {
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - '" + key<%=cid %> + "' rename failed. " + ftp_<%=cid %>.getReplyString());
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename fail.");
}
}catch(IOException e){
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename fail.");
<%
@@ -143,21 +94,100 @@ try{
}
}
<%
}
%>
} catch (Exception e) {
} else {%>//normal ftp
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
String tmp_<%=cid%> = pid + System.currentTimeMillis();
try{
for (String key<%=cid %> : keySet<%=cid %>) {
<%if(("always").equals(overwrite)){%>
String fromFile_<%=cid%> = key<%=cid %>;
String toFile_<%=cid%> = map<%=cid %>.get(key<%=cid %>);
if (!fromFile_<%=cid%>.equals(toFile_<%=cid%>)){
if(ftp_<%=cid%>.exists(fromFile_<%=cid%>)){
if(ftp_<%=cid%>.exists(toFile_<%=cid%> + "." + tmp_<%=cid%>)){
ftp_<%=cid%>.delete(toFile_<%=cid%> + "." + tmp_<%=cid%>);
}
if(ftp_<%=cid%>.exists(toFile_<%=cid%>)){
ftp_<%=cid %>.rename(toFile_<%=cid%>, toFile_<%=cid%> + "." + tmp_<%=cid%>);
}
ftp_<%=cid %>.rename(fromFile_<%=cid%>, toFile_<%=cid%>);
if(ftp_<%=cid%>.exists(toFile_<%=cid%> + "." + tmp_<%=cid%>)){
ftp_<%=cid%>.delete(toFile_<%=cid%> + "." + tmp_<%=cid%>);
}
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - '" + toFile_<%=cid%> + "' rename ok");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
nb_file_<%=cid%>++;
}
}
<%}%>
<%if(("never").equals(overwrite)){%>
if(!(ftp_<%=cid%>.exists(map<%=cid %>.get(key<%=cid %>)))) {
ftp_<%=cid %>.rename(key<%=cid %>, map<%=cid %>.get(key<%=cid %>));
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - '" + key<%=cid %> + "' rename ok");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
nb_file_<%=cid%>++;
}
<%}%>
<%if(("size_differ").equals(overwrite)){%>
if((ftp_<%=cid%>.exists(map<%=cid %>.get(key<%=cid %>)))){
com.enterprisedt.net.ftp.FTPFile ftpnewNamefile<%=cid%>=ftp_<%=cid%>.fileDetails(map<%=cid %>.get(key<%=cid %>));
com.enterprisedt.net.ftp.FTPFile ftporigNamefile<%=cid%>=ftp_<%=cid%>.fileDetails(key<%=cid %>);
long newNameSize<%=cid%>=ftpnewNamefile<%=cid%>.size();
long origNameSize<%=cid%>=ftporigNamefile<%=cid%>.size();
if(!(newNameSize<%=cid%>==origNameSize<%=cid%>)) {
String fromFile_<%=cid%> = key<%=cid %>;
String toFile_<%=cid%> = map<%=cid %>.get(key<%=cid %>);
if(ftp_<%=cid%>.exists(fromFile_<%=cid%>)){
if(ftp_<%=cid%>.exists(toFile_<%=cid%> + "." + tmp_<%=cid%>)){
ftp_<%=cid%>.delete(toFile_<%=cid%> + "." + tmp_<%=cid%>);
}
if(ftp_<%=cid%>.exists(toFile_<%=cid%>)){
ftp_<%=cid %>.rename(toFile_<%=cid%>, toFile_<%=cid%> + "." + tmp_<%=cid%>);
}
ftp_<%=cid %>.rename(fromFile_<%=cid%>, toFile_<%=cid%>);
if(ftp_<%=cid%>.exists(toFile_<%=cid%> + "." + tmp_<%=cid%>)){
ftp_<%=cid%>.delete(toFile_<%=cid%> + "." + tmp_<%=cid%>);
}
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - '" + toFile_<%=cid%> + "' rename ok");
<%}%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
nb_file_<%=cid%>++;
}
}
}else{
ftp_<%=cid %>.rename(key<%=cid %>, map<%=cid %>.get(key<%=cid %>));
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
nb_file_<%=cid%>++;
}
<%}%>
}
}catch(com.enterprisedt.net.ftp.FTPException e){
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename fail.");
throw e;
}
<%}%>
}catch(java.lang.Exception e){
<%
if(("true").equals(dieOnError)){
if(("true").equals(dieOnError)){
%>
throw(e);
<%
}else{
}else{
%>
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - " + e.getMessage());
<%}%>
System.err.print(e.getMessage());
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - " + e.getMessage());
<%}%>
System.err.print(e.getMessage());
<%
}
}
%>
}

View File

@@ -55,8 +55,6 @@ if (("true").equals(useExistingConn)) {
} else {
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
%>
<%
}
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
@@ -133,7 +131,7 @@ if (sftp) {// *** sftp *** //
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
<%if(use_encoding) {%>
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
@@ -169,76 +167,179 @@ if (sftp) {// *** sftp *** //
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
<%
} else {
if (ftps) { // *** ftps *** //
} else if (ftps) { // *** ftps *** //
/* creating connection */
if (("false").equals(useExistingConn)) {
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
<%
/* creating connection */
if (("false").equals(useExistingConn)) {
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
<%
} else {
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
<%
if (moveToCurrentDir) {
%>
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(<%=remotedir %>);
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
if (!cwdSuccess_<%=cid %>) {
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
}
<%
}
}
} else { // *** ftp *** //
%>
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
<%
if (("true").equals(useExistingConn)) {
%>
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%
if (moveToCurrentDir) {
%>
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(<%=remotedir %>);
try {
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
if (isHTTPProxyUsed) {
String httpProxyHost = System.getProperty("http.proxyHost");
int httpProxyPort = Integer.getInteger("http.proxyPort");
String httpProxyUser = System.getProperty("http.proxyUser");
String httpProxyPass = System.getProperty("http.proxyPassword");
if (!cwdSuccess_<%=cid %>) {
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
}
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
} else {
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
}
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
<%
}
} else {
passwordFieldName = "__PASSWORD__";
if(isLog4jEnabled){
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%
}
} //common code for ftp and ftps:
}
passwordFieldName = "__PASSWORD__";
%>
<% /*store real pwd*/ %>
String rootWorkDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
if (!isLoginSuccessful_<%=cid %>) {
throw new RuntimeException("Login failed");
}
<%
for (int i = 0; i < files.size(); i++) {
Map<String, String> line = files.get(i);
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
%>
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,"");
list<%=cid %>.add(map<%=cid %><%=i %>);
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
<%
}
}
if (protectionLevel != null && !protectionLevel.isEmpty()) {
%>
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) { <% /* start main loop */ %>
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
<%
}
}
%>
} catch (Exception e) {
<%if(isLog4jEnabled){%>
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
<%}%>
throw e;
}
<%
if ("PASSIVE".equals(connectMode)) {
%>
ftp_<%=cid %>.enterLocalPassiveMode();
<%
if (isLog4jEnabled) {
%>
log.debug("<%=cid%> - Using the passive mode.");
<%
}
}
} else {
%>
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
<%
if (moveToCurrentDir) {
%>
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(<%=remotedir %>);
if (!cwdSuccess_<%=cid %>) {
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
}
<%
}
}
%>
<% /*store real pwd*/ %>
String rootWorkDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
<%
for (int i = 0; i < files.size(); i++) {
Map<String, String> line = files.get(i);
%>
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,"");
list<%=cid %>.add(map<%=cid %><%=i %>);
<%
}
%>
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) { <% /* start main loop */ %>
<%
} else {// *** ftp *** //
%>
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> = null;
<%if (("true").equals(useExistingConn)) {%>
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
<%if(isLog4jEnabled){%>
if(ftp_<%=cid %>!=null) {
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
}
<%}%>
<%if(!moveToCurrentDir){%>
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
<%}%>
<%} else {%>
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
ftp_<%=cid %>.setRemoteHost(<%=host %>);
ftp_<%=cid %>.setRemotePort(<%=port %>);
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
<%}%>
<% if (("ACTIVE").equals(connectMode)) {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
<%} else {%>
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
<%}%>
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
<%}%>
ftp_<%=cid %>.connect();
<%
passwordFieldName = "__PASSWORD__";
%>
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
<%}%>
<%}%>
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
<%for (int i = 0; i < files.size(); i++) {
Map<String, String> line = files.get(i);
%>
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,"");
list<%=cid %>.add(map<%=cid %><%=i %>);
<%}%>
String remotedir<%=cid %> = <%=remotedir%>;
ftp_<%=cid %>.chdir(remotedir<%=cid %>);
String root<%=cid %> = ftp_<%=cid %>.pwd();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Truncating file from the server.");
<%}%>
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
<%}%>

View File

@@ -67,7 +67,7 @@
}
} else {// *** ftp *** //
%>
} <% /* close main loop */ %>
}
<%
if(!("true").equals(useExistingConn)){
if(("true").equals(ignoreFailureAtQuit)){
@@ -76,13 +76,13 @@
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
}catch(java.io.IOException ioe_<%=cid%>){
}catch(java.net.SocketException se_<%=cid%>){
<%if(isLog4jEnabled){%>
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
<%}%>
//ignore failure
}
@@ -92,12 +92,18 @@
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Closing the connection to the server.");
<%}%>
ftp_<%=cid %>.disconnect();
ftp_<%=cid %>.quit();
<%if(isLog4jEnabled){%>
log.info("<%=cid%> - Connection to the server closed.");
<%}%>
<%
}
}else{
if(!moveToCurrentDir){
%>
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
<%
}
}
}
%>

View File

@@ -149,7 +149,7 @@ COMPATIBILITY="ALL"
<DEFAULT>"."</DEFAULT>
</PARAMETER>
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="31" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="31" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
<DEFAULT>true</DEFAULT>
</PARAMETER>
@@ -249,13 +249,13 @@ COMPATIBILITY="ALL"
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
<IMPORT NAME="jakarta-oro" MODULE="jakarta-oro-2.0.8.jar" MVN="mvn:org.talend.libraries/jakarta-oro-2.0.8/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache/lib/jakarta-oro-2.0.8.jar" REQUIRED="true" />
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
</IMPORTS>
</CODEGENERATION>
<RETURNS>

View File

@@ -73,10 +73,10 @@
}
}
<%
} else { /* ftp+s */
} else if (ftps) { /* ftps */
%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file truncated.");
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
for (String key<%=cid %> : keySet<%=cid %>) {
<% /*prepare filemask*/ %>
String filemask<%=cid %> = key<%=cid %>;
@@ -188,4 +188,44 @@
<% /*return to root dir for next filemask*/ %>
ftp_<%=cid %>.changeWorkingDirectory(rootWorkDir_<%=cid %>);
<%} %>
<%
} else { /* ftp */
%>
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file truncated.");
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
try {
for (String key<%=cid %> : keySet<%=cid %>) {
String filemask<%=cid %> = key<%=cid %>;
String dir<%=cid %> = null;
<%if(!useGlob) {//perl5 mode not support windows(\) path separator at the mask string%>
String mask<%=cid %> = filemask<%=cid %>.replaceAll("\\\\", "/") ;
<%}else{%>
String mask<%=cid %> = filemask<%=cid %>;
<%}%>
int i<%=cid %> = mask<%=cid %>.lastIndexOf('/');
if (i<%=cid %>!=-1) {
dir<%=cid %> = mask<%=cid %>.substring(0, i<%=cid %>);
mask<%=cid %> = mask<%=cid %>.substring(i<%=cid %>+1);
}
if (dir<%=cid %>!=null && !"".equals(dir<%=cid %>)) ftp_<%=cid %>.chdir(dir<%=cid %>);
<%if(!useGlob) {%>
mask<%=cid %> = org.apache.oro.text.GlobCompiler.globToPerl5(mask<%=cid %>.toCharArray(), org.apache.oro.text.GlobCompiler.DEFAULT_MASK);
<%}%>
String[] listings<%=cid %> = ftp_<%=cid %>.dir(null, false);
for (int m<%=cid %> = 0; m<%=cid %> < listings<%=cid %>.length; m<%=cid %>++) {
if (listings<%=cid %>[m<%=cid %>].matches(mask<%=cid %>)) {
ftp_<%=cid %>.put(new ByteArrayInputStream(new byte[]{}), listings<%=cid %>[m<%=cid %>]);
globalMap.put("<%=cid %>_CURRENT_STATUS", "File truncate OK.");
<%if(isLog4jEnabled){%>
log.debug("<%=cid%> - '" + listings<%=cid %>[m<%=cid %>] + "' truncate ok.");
<%}%>
nb_file_<%=cid%>++;
}
}
}
}catch(com.enterprisedt.net.ftp.FTPException e){
globalMap.put("<%=cid %>_CURRENT_STATUS", "File truncate fail.");
throw e;
}
ftp_<%=cid %>.chdir(root<%=cid %>);
<%}%>

View File

@@ -131,7 +131,7 @@
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="commons-compress" MODULE="commons-compress-1.19.jar" MVN="mvn:org.apache.commons/commons-compress/1.19" REQUIRED="true" />
<IMPORT NAME="COMMONS-COMPRESS-1.10" MODULE="commons-compress-1.10.jar" MVN="mvn:org.talend.libraries/commons-compress-1.10/6.1.0" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-compress-1.10.jar" REQUIRED="true" />
<IMPORT NAME="Encrypt-Zip" MODULE="checkArchive-1.1-20181130.jar" MVN="mvn:org.talend.libraries/checkArchive-1.1-20181130/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/checkArchive-1.1-20181130.jar" REQUIRED="true" />
<IMPORT NAME="zip4j_1.3.1.jar" MODULE="zip4j_1.3.1.jar" MVN="mvn:org.talend.libraries/zip4j_1.3.1/6.0.0" REQUIRED="true" />
<IMPORT NAME="talendzip-1.0-20190527.jar" MODULE="talendzip-1.0-20190527.jar" MVN="mvn:org.talend.libraries/talendzip/1.0-20190527" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talendzip-1.0-20190527.jar" REQUIRED="true" />

View File

@@ -464,7 +464,6 @@
log.error("<%=cid%> - " +e.getMessage());
<%}%>
System.err.println(e.getMessage());
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
<% } %>
}
java.util.zip.ZipEntry entry_<%=cid %> = null;
@@ -480,7 +479,6 @@
log.error("<%=cid%> - " +e.getMessage());
<%}%>
System.err.println(e.getMessage());
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
break;
<% } %>
}
@@ -505,7 +503,6 @@
log.error("<%=cid%> - " +e.getMessage());
<%}%>
System.err.println(e.getMessage());
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
<% } %>
}
<%
@@ -531,7 +528,6 @@
log.error("<%=cid%> - " +e.getMessage());
<%}%>
System.err.println(e.getMessage());
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
<% } %>
}
<%
@@ -610,9 +606,7 @@
} else if(rejectConnName.equals(firstConnName)){%>
<%=rejectConnName%>.errorMessage = e.getMessage() + " - Line: " + tos_count_<%=node.getUniqueName() %>;
<%
} %>
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
<%
}
}
%>
}
@@ -1082,7 +1076,6 @@
log.error("<%=cid%> - " + e.getMessage());
<%}%>
System.err.println(e.getMessage());
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
<% } %>
}
java.util.zip.ZipEntry entry_<%=cid %> = null;
@@ -1098,7 +1091,6 @@
log.error("<%=cid%> - " + e.getMessage());
<%}%>
System.err.println(e.getMessage());
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
break;
<% } %>
}
@@ -1293,7 +1285,6 @@
log.error("<%=cid%> - " +e.getMessage());
<%}%>
System.err.println(e.getMessage());
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
<% } %>
}//TD110 end
@@ -1397,9 +1388,7 @@
%>
<%=rejectConnName%>.errorMessage = e.getMessage() + " - Line: " + tos_count_<%=node.getUniqueName() %>;
<%
} %>
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
<%
}
}
%>
}

View File

@@ -213,7 +213,7 @@
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Talen File Enhanced"
MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar" REQUIRED="true" />
MODULE="talend_file_enhanced_20070724.jar" MVN="mvn:org.talend.libraries/talend_file_enhanced_20070724/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced_20070724.jar" REQUIRED="true" />
<IMPORT NAME="Talend_CSV" MODULE="talendcsv.jar" MVN="mvn:org.talend.libraries/talendcsv/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.csv/lib/talendcsv.jar"
REQUIRED="true" />
</IMPORTS>

View File

@@ -185,7 +185,7 @@
<IMPORT NAME="simpleexcel" MODULE="simpleexcel-2.1-20190507.jar" MVN="mvn:org.talend.libraries/simpleexcel-2.1-20190507/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.excel/lib/simpleexcel-2.1-20190507.jar" REQUIRED_IF="(VERSION_2007 == 'true') AND GENERATION_MODE == 'EVENT_MODE')" />
<IMPORT NAME="commons-collections4" MODULE="commons-collections4-4.1.jar" MVN="mvn:org.talend.libraries/commons-collections4-4.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.excel/lib/commons-collections4-4.1.jar" REQUIRED_IF="(VERSION_2007 == 'true')" />
<IMPORT NAME="commons-compress" MODULE="commons-compress-1.19.jar" MVN="mvn:org.apache.commons/commons-compress/1.19" REQUIRED_IF="(VERSION_2007 == 'true')" />
<IMPORT NAME="commons-compress" MODULE="commons-compress-1.18.jar" MVN="mvn:org.apache.commons/commons-compress/1.18" REQUIRED_IF="(VERSION_2007 == 'true')" />
<IMPORT NAME="commons-math3" MODULE="commons-math3-3.6.1.jar" MVN="mvn:org.apache.commons/commons-math3/3.6.1" REQUIRED_IF="(VERSION_2007 == 'true')" />
</IMPORTS>
</CODEGENERATION>

View File

@@ -131,7 +131,7 @@
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Talend File Enhanced" MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar" REQUIRED="true" />
<IMPORT NAME="Talend File Enhanced" MODULE="talend_file_enhanced_20070724.jar" MVN="mvn:org.talend.libraries/talend_file_enhanced_20070724/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced_20070724.jar" REQUIRED="true" />
</IMPORTS>
</CODEGENERATION>

View File

@@ -165,7 +165,7 @@
REQUIRED_IF="(READ_BY == 'XPATH')" BundleID="" />
<IMPORT NAME="Java_JAXEN1.1" MODULE="jaxen-1.1.1.jar" MVN="mvn:org.talend.libraries/jaxen-1.1.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.dom4j-jaxen/lib/jaxen-1.1.1.jar"
REQUIRED_IF="(READ_BY == 'XPATH')" BundleID="" />
<IMPORT NAME="json-lib" MODULE="json-lib-2.4.5-talend.jar" MVN="mvn:net.sf.json-lib/json-lib/2.4.5-talend" REQUIRED_IF="(READ_BY == 'XPATH')" />
<IMPORT NAME="json-lib" MODULE="json-lib-2.4.2-talend.jar" MVN="mvn:net.sf.json-lib/json-lib/2.4.2-talend" REQUIRED_IF="(READ_BY == 'XPATH')" />
<IMPORT NAME="commons_lang" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar"
REQUIRED_IF="(READ_BY == 'XPATH')" />
<IMPORT NAME="commons_logging" MODULE="commons-logging-1.1.1.jar" MVN="mvn:org.talend.libraries/commons-logging-1.1.1/6.0.0" UrlPath="platform:/base/plugins/org.apache.commons.logging_1.1.1.v201101211721.jar"

View File

@@ -118,7 +118,7 @@
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Talen File Enhanced"
MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar" REQUIRED="true" />
MODULE="talend_file_enhanced_20070724.jar" MVN="mvn:org.talend.libraries/talend_file_enhanced_20070724/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced_20070724.jar" REQUIRED="true" />
<IMPORT NAME="Talend_CSV" MODULE="talendcsv.jar" MVN="mvn:org.talend.libraries/talendcsv/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.csv/lib/talendcsv.jar" REQUIRED="true" />
</IMPORTS>
</CODEGENERATION>

View File

@@ -81,11 +81,11 @@
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Talen File Enhanced" MODULE="talend_file_enhanced-1.1.jar" REQUIRED="true" />
<IMPORT NAME="Talen File Enhanced" MODULE="talend_file_enhanced_20070724.jar" REQUIRED="true" />
</IMPORTS>
</CODEGENERATION>
<RETURNS>
<RETURN NAME="NB_LINE" TYPE="id_Integer" AVAILABILITY="AFTER" />
</RETURNS>
</COMPONENT>
</COMPONENT>

View File

@@ -39,7 +39,7 @@
String skipEmptyRows = "true"; //("true").equals(ElementParameterParser.getValue(node, "__SKIP_EMPTY_ROWS__"));
boolean trimAll = ("true").equals(ElementParameterParser.getValue(node,"__TRIMALL__"));
String encoding = ElementParameterParser.getValue(node,"__ENCODING__");
//need to process rows longger than 100,000 characters, the property SafetySwitch(in talend_file_enhanced-1.1.jar) should be sent to false.(the default is true)
//need to process rows longger than 100,000 characters, the property SafetySwitch(in talend_file_enhanced_20070724.jar) should be sent to false.(the default is true)
//that means if check the option(true), the logic value of bSafetySwitch should be changed to false (negate the property)
boolean bSafetySwitch = !(("true").equals(ElementParameterParser.getValue(node, "__PROCESS_LONG_ROW__")));
String advancedSeparatorStr = ElementParameterParser.getValue(node, "__ADVANCED_SEPARATOR__");

View File

@@ -128,7 +128,7 @@
<CODEGENERATION>
<IMPORTS>
<IMPORT NAME="Talen File Enhanced" MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar" REQUIRED="true" />
<IMPORT NAME="Talen File Enhanced" MODULE="talend_file_enhanced_20070724.jar" MVN="mvn:org.talend.libraries/talend_file_enhanced_20070724/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced_20070724.jar" REQUIRED="true" />
</IMPORTS>
</CODEGENERATION>

Some files were not shown because too many files have changed in this diff Show More