Compare commits
2 Commits
mbasiuk/TP
...
patch/TPS-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
20bf94ec21 | ||
|
|
fd60c1edca |
@@ -3,15 +3,17 @@ version: 7.2.1
|
||||
module: https://talend.poolparty.biz/coretaxonomy/42
|
||||
product:
|
||||
- https://talend.poolparty.biz/coretaxonomy/23
|
||||
- https://talend.poolparty.biz/coretaxonomy/26
|
||||
- https://talend.poolparty.biz/coretaxonomy/24
|
||||
---
|
||||
|
||||
# TPS-4808
|
||||
# TPS-3403
|
||||
|
||||
| Info | Value |
|
||||
| ---------------- | ---------------- |
|
||||
| Patch Name | Patch\_20210521\_TPS-4808\_v1-7.2.1 |
|
||||
| Release Date | 2021-05-21 |
|
||||
| Target Version | 20190620_1446-V7.2.1 |
|
||||
| Patch Name | Patch\_20190907\_TPS-3403\_v1-7.2.1 |
|
||||
| Release Date | 2019-09-07 |
|
||||
| Target Version | 20190620\_1446-V7.2.1 |
|
||||
| Product affected | Talend Studio |
|
||||
|
||||
## Introduction
|
||||
@@ -21,24 +23,21 @@ This is a self-contained patch.
|
||||
**NOTE**: For information on how to obtain this patch, reach out to your Support contact at Talend.
|
||||
|
||||
## Fixed issues
|
||||
This patch contains this following fixe:
|
||||
|
||||
This patch contains the following fixes:
|
||||
|
||||
- TPS-4808 [7.2.1] tSoap components doesn't work and returning a 400 Bad request message(TDI-44893)
|
||||
- TPS-3403 filecopy fail with java11 (TDI-42230)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Consider the following requirements for your system:
|
||||
|
||||
- Talend Studio 7.2.1 must be installed.
|
||||
|
||||
## Installation
|
||||
|
||||
**NOTE**: If the patch is deployed in the approach Installing the patch using Talend Studio, the folder configuration under this patch must be replaced manually.
|
||||
## Installation
|
||||
|
||||
### Installing the patch using Software update
|
||||
|
||||
1) Logon TAC and switch to Configuration->Software Update, then enter the correct values and save referring to the documentation: https://help.talend.com/reader/f7Em9WV_cPm2RRywucSN0Q/j9x5iXV~vyxMlUafnDejaQ
|
||||
1) Logon TAC and switch to Configuration->Software Update, then enter the correct values and save referring to the documentation: https://help.talend.com/reader/f7Em9WV\_cPm2RRywucSN0Q/j9x5iXV~vyxMlUafnDejaQ
|
||||
|
||||
2) Switch to Software update page, where the new patch will be listed. The patch can be downloaded from here into the nexus repository.
|
||||
|
||||
@@ -54,6 +53,14 @@ Consider the following requirements for your system:
|
||||
|
||||
Execute the following commands:
|
||||
|
||||
1. Talend-Studio-win-x86_64.exe -nosplash -application org.talend.commandline.CommandLine -consoleLog -data commandline-workspace startServer -p 8002 --talendDebug
|
||||
2. initRemote {tac_url} -ul {TAC login username} -up {TAC login password}
|
||||
1. Talend-Studio-win-x86\_64.exe -nosplash -application org.talend.commandline.CommandLine -consoleLog -data commandline-workspace startServer -p 8002 --talendDebug
|
||||
2. initRemote {tac\_url} -ul {TAC login username} -up {TAC login password}
|
||||
3. checkAndUpdate -tu {TAC login username} -tup {TAC login password}
|
||||
|
||||
## Affected files for this patch
|
||||
|
||||
The following files are installed by this patch:
|
||||
|
||||
- plugins/org.talend.designer.components.localprovider\_7.2.1.20190614\_0309/components/tFileCopy/tFileCopy\_java.xml
|
||||
- plugins/org.talend.libraries.custom\_7.2.1.20181114\_0551/lib/filecopy-2.0.0.jar
|
||||
|
||||
|
||||
@@ -106,21 +106,6 @@
|
||||
exist_tSQLDB = true;
|
||||
}
|
||||
|
||||
|
||||
String[] tcompV0Components = new String[] {
|
||||
"tAzureStorageConnection",
|
||||
"tGoogleDriveConnection",
|
||||
"tMarketoConnection",
|
||||
"tSnowflakeConnection",
|
||||
"tSalesforceConnection",
|
||||
"tNetsuiteConnection",
|
||||
"tMarkLogicConnection"
|
||||
};
|
||||
List<INode> tcompV0ComponentsList = new ArrayList<INode>();
|
||||
for (String tcompV0Component : tcompV0Components) {
|
||||
tcompV0ComponentsList.addAll((List<INode>)process.getNodesOfType(tcompV0Component));
|
||||
}
|
||||
|
||||
boolean exist_tSCP = false;
|
||||
List<INode> scpComponentsList = (List<INode>)process.getNodesOfType("tSCPConnection");
|
||||
if (scpComponentsList.size() > 0) {
|
||||
@@ -180,8 +165,6 @@
|
||||
exist_tSalesforce = true;
|
||||
}
|
||||
|
||||
List<INode> snowflakeComponentsList = (List<INode>)process.getNodesOfType("tSnowflakeConnection");
|
||||
|
||||
boolean exist_tSAP= false;
|
||||
List<INode> sapComponentsList = (List<INode>)process.getNodesOfType("tSAPConnection");
|
||||
if (sapComponentsList.size() > 0) {
|
||||
@@ -462,7 +445,6 @@
|
||||
lastStr = "";
|
||||
}
|
||||
}
|
||||
enableLogStash = "true".equalsIgnoreCase(System.getProperty("monitoring"));
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(!"".equals(log4jLevel)){
|
||||
@@ -511,13 +493,7 @@
|
||||
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("appender.file.maxsize", "52428800");
|
||||
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("appender.file.maxbackup", "20");
|
||||
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("host", "false");
|
||||
|
||||
System.getProperties().stringPropertyNames().stream()
|
||||
.filter(it -> it.startsWith("monitoring.audit.logger.properties."))
|
||||
.forEach(key -> properties_<%=jobCatcherNode.getUniqueName()%>.setProperty(key.substring("monitoring.audit.logger.properties.".length()), System.getProperty(key)));
|
||||
|
||||
org.apache.log4j.Logger.getLogger("audit").setLevel(org.apache.log4j.Level.DEBUG);
|
||||
|
||||
|
||||
auditLogger_<%=jobCatcherNode.getUniqueName()%> = org.talend.job.audit.JobEventAuditLoggerFactory.createJobAuditLogger(properties_<%=jobCatcherNode.getUniqueName()%>);
|
||||
}
|
||||
<%
|
||||
@@ -608,16 +584,7 @@
|
||||
} //endIf
|
||||
%>
|
||||
context.setContextType("<%=ctxParam.getName()%>", "<%=ctxParam.getType()%>");
|
||||
if(context.getStringValue("<%=ctxParam.getName()%>") == null) {
|
||||
context.<%=ctxParam.getName()%> = null;
|
||||
} else {
|
||||
<%//}B1 block%>
|
||||
<%
|
||||
String typeToGenerate ="String";
|
||||
if( !(ctxParam.getType().equals("id_File") || ctxParam.getType().equals("id_Directory") ||ctxParam.getType().equals("id_List Of Value"))) {
|
||||
typeToGenerate=JavaTypesManager.getTypeToGenerate(ctxParam.getType(),true);
|
||||
}
|
||||
|
||||
if (ctxParam.getType().equals("id_Password")) {
|
||||
%>
|
||||
String pwd_<%=ctxParam.getName()%>_value = context.getProperty("<%=ctxParam.getName()%>");
|
||||
@@ -635,7 +602,13 @@
|
||||
}
|
||||
}
|
||||
<%
|
||||
} else if(typeToGenerate.equals("java.util.Date")) {
|
||||
continue;
|
||||
}
|
||||
String typeToGenerate ="String";
|
||||
if( !(ctxParam.getType().equals("id_File") || ctxParam.getType().equals("id_Directory") ||ctxParam.getType().equals("id_List Of Value"))) {
|
||||
typeToGenerate=JavaTypesManager.getTypeToGenerate(ctxParam.getType(),true);
|
||||
}
|
||||
if(typeToGenerate.equals("java.util.Date")) {
|
||||
%>
|
||||
try{
|
||||
String context_<%=ctxParam.getName()%>_value = context.getProperty("<%=ctxParam.getName()%>");
|
||||
@@ -694,10 +667,6 @@
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%//{B1 block%>
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} <%/*close last method*/%>
|
||||
@@ -1290,11 +1259,13 @@ if (execStat) {
|
||||
} else {
|
||||
type = "ftp";
|
||||
}
|
||||
if ("sftp".equals(type)) { %>
|
||||
if ("ftp".equals(type)) { %>
|
||||
((com.enterprisedt.net.ftp.FTPClient) obj_conn).quit();
|
||||
<%} else if ("sftp".equals(type)) { %>
|
||||
((com.jcraft.jsch.ChannelSftp) obj_conn).quit();
|
||||
<%} else { %>
|
||||
((org.apache.commons.net.ftp.FTPClient) obj_conn).disconnect();
|
||||
<%}%>
|
||||
<%} else { %>
|
||||
((org.apache.commons.net.ftp.FTPSClient) obj_conn).disconnect();
|
||||
<%}%>
|
||||
}
|
||||
<% } %>
|
||||
} catch (java.lang.Exception e) {
|
||||
@@ -1469,14 +1440,12 @@ if (execStat) {
|
||||
<% } %>
|
||||
<% for (INode existNode : existComponentsList) { %>
|
||||
connections.put("col_<%=existNode.getUniqueName()%>", globalMap.get("col_<%=existNode.getUniqueName()%>"));
|
||||
<% }
|
||||
<% } %>
|
||||
|
||||
for (INode tcompV0ConnectionNode : tcompV0ComponentsList) { %>
|
||||
connections.put("<%=tcompV0ConnectionNode.getUniqueName()%>_connection", globalMap.get("<%=tcompV0ConnectionNode.getUniqueName()%>_connection"));
|
||||
connections.put("<%=tcompV0ConnectionNode.getUniqueName()%>_COMPONENT_RUNTIME_PROPERTIES", globalMap.get("<%=tcompV0ConnectionNode.getUniqueName()%>_COMPONENT_RUNTIME_PROPERTIES"));
|
||||
<% }
|
||||
|
||||
for (INode sapNode : sapComponentsList) { %>
|
||||
<% for (INode salesforceNode : salesforceComponentsList) { %>
|
||||
connections.put("conn_<%=salesforceNode.getUniqueName()%>", globalMap.get("conn_<%=salesforceNode.getUniqueName()%>"));
|
||||
<% } %>
|
||||
<% for (INode sapNode : sapComponentsList) { %>
|
||||
connections.put("conn_<%=sapNode.getUniqueName()%>", globalMap.get("conn_<%=sapNode.getUniqueName()%>"));
|
||||
<% } %>
|
||||
<% for (INode gsNode : gsComponentsList) { %>
|
||||
@@ -1540,10 +1509,12 @@ if (execStat) {
|
||||
}
|
||||
} else if (arg.startsWith("--log4jLevel=")) {
|
||||
log4jLevel = arg.substring(13);
|
||||
} else if (arg.startsWith("--monitoring") && arg.contains("=")) {//for trunjob call
|
||||
final int equal = arg.indexOf('=');
|
||||
final String key = arg.substring("--".length(), equal);
|
||||
System.setProperty(key, arg.substring(equal + 1));
|
||||
} else if (arg.startsWith("--monitoring=")) {//for trunjob call
|
||||
enableLogStash = "true".equalsIgnoreCase(arg.substring(13));
|
||||
}
|
||||
|
||||
if(!enableLogStash) {
|
||||
enableLogStash = "true".equalsIgnoreCase(System.getProperty("monitoring"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -62,8 +62,6 @@ class IndexedRecordToRowStructGenerator {
|
||||
%>
|
||||
boolean <%=codeVarIsDynamicInitialized%> = false;
|
||||
routines.system.Dynamic <%=codeVarDynamic%> = new routines.system.Dynamic();
|
||||
//Workaround for TDI-44051, TcimpV0 do not need DbmsID
|
||||
<%=codeVarDynamic%>.setDbmsId("<%=cid%>");
|
||||
<%
|
||||
}
|
||||
|
||||
@@ -117,11 +115,10 @@ class IndexedRecordToRowStructGenerator {
|
||||
for (org.apache.avro.Schema.Field dynamicField_<%=cid%> : dynSchema_<%=cid%>.getFields()){
|
||||
routines.system.DynamicMetadata dynamicMetadata_<%=cid%> = new routines.system.DynamicMetadata();
|
||||
org.apache.avro.Schema dynamicFieldSchema_<%=cid%> = dynamicField_<%=cid%>.schema();
|
||||
String dbName = dynamicField_<%=cid%>.getProp("talend.field.dbColumnName");
|
||||
// set name
|
||||
dynamicMetadata_<%=cid%>.setName(dynamicField_<%=cid%>.name());
|
||||
// set db name
|
||||
dynamicMetadata_<%=cid%>.setDbName(dbName==null?dynamicField_<%=cid%>.name():dbName);
|
||||
dynamicMetadata_<%=cid%>.setDbName(dynamicField_<%=cid%>.name());
|
||||
// set nullable
|
||||
if (org.talend.daikon.avro.AvroUtils.isNullable(dynamicFieldSchema_<%=cid%>)) {
|
||||
dynamicMetadata_<%=cid%>.setNullable(true);
|
||||
|
||||
@@ -289,15 +289,6 @@ if(isRunInMultiThread){
|
||||
}
|
||||
<% } %>
|
||||
}
|
||||
|
||||
//if the stored or passed value is "<TALEND_NULL>" string, it mean null
|
||||
public String getStringValue(String key) {
|
||||
String origin_value = this.getProperty(key);
|
||||
if(NULL_VALUE_EXPRESSION_IN_COMMAND_STRING_FOR_CHILD_JOB_ONLY.equals(origin_value)) {
|
||||
return null;
|
||||
}
|
||||
return origin_value;
|
||||
}
|
||||
|
||||
<%
|
||||
for (IContextParameter ctxParam :params)
|
||||
|
||||
@@ -66,7 +66,6 @@ import org.talend.designer.codegen.model.CodeGeneratorEmittersPoolFactory;
|
||||
import org.talend.designer.codegen.model.CodeGeneratorInternalTemplatesFactoryProvider;
|
||||
import org.talend.designer.codegen.proxy.JetProxy;
|
||||
import org.talend.designer.core.generic.model.Component;
|
||||
import org.talend.designer.runprocess.ProcessorUtilities;
|
||||
|
||||
/**
|
||||
* CodeGenerator.
|
||||
@@ -354,12 +353,8 @@ public class CodeGenerator implements ICodeGenerator {
|
||||
.getDefault().getService(IResourcesDependenciesService.class);
|
||||
String resourcePathForContext = null;
|
||||
if (process instanceof IProcess2) {
|
||||
String value = contextPar.getValue();
|
||||
// for runtime
|
||||
if (!ProcessorUtilities.isExportConfig()) {
|
||||
contextPar.setType(JavaTypesManager.FILE.getId());
|
||||
}
|
||||
resourcePathForContext = resourceService.getResourcePathForContext(process, value);
|
||||
resourcePathForContext = resourceService.getResourcePathForContext(process,
|
||||
contextPar.getValue());
|
||||
} else {
|
||||
// for PreviewFileInputContentDataProcess run
|
||||
resourcePathForContext = resourceService.getResourceItemFilePath(contextPar.getValue());
|
||||
|
||||
@@ -94,7 +94,7 @@ public class JavaRoutineSynchronizer extends AbstractRoutineSynchronizer {
|
||||
|
||||
private void syncRoutineItems(Collection<RoutineItem> routineObjects, boolean forceUpdate) throws SystemException {
|
||||
for (RoutineItem routineItem : routineObjects) {
|
||||
syncRoutine(routineItem, true, forceUpdate);
|
||||
syncRoutine(routineItem, true, true, forceUpdate);
|
||||
}
|
||||
syncSystemRoutine(ProjectManager.getInstance().getCurrentProject());
|
||||
}
|
||||
|
||||
@@ -28,9 +28,9 @@ import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.ResourceBundle;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
@@ -70,6 +70,7 @@ import org.talend.core.model.components.ComponentCategory;
|
||||
import org.talend.core.model.components.ComponentManager;
|
||||
import org.talend.core.model.components.ComponentProviderInfo;
|
||||
import org.talend.core.model.components.ComponentUtilities;
|
||||
import org.talend.core.model.components.EComponentType;
|
||||
import org.talend.core.model.components.IComponent;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.model.components.IComponentsHandler;
|
||||
@@ -837,21 +838,23 @@ public class ComponentsFactory implements IComponentsFactory {
|
||||
init(false);
|
||||
}
|
||||
|
||||
// check if reference joblet component presents
|
||||
JobletUtil jobletUtils = new JobletUtil();
|
||||
Optional<IComponent> result = jobletUtils.findComponentByName(componentList, name, paletteType);
|
||||
if (!result.isPresent()) {
|
||||
// check if any name matching joblet component presents
|
||||
if (jobletUtils.matchExpression(name)) {
|
||||
name = StringUtils.substringAfterLast(name, ":"); //$NON-NLS-1$
|
||||
if (StringUtils.isNotBlank(name)) {
|
||||
result = jobletUtils.findComponentByName(componentList, name, paletteType);
|
||||
for (IComponent comp : componentList) {
|
||||
if (comp.getComponentType() != EComponentType.JOBLET) {
|
||||
continue;
|
||||
}
|
||||
String comName = comp.getName();
|
||||
if (comp != null && paletteType.equals(comp.getPaletteType())) {
|
||||
if (comName.equals(name)) {
|
||||
return comp;
|
||||
} else if (new JobletUtil().matchExpression(comName)) {
|
||||
String[] names = comName.split(":"); //$NON-NLS-1$
|
||||
comName = names[1];
|
||||
if (comName.equals(name)) {
|
||||
return comp;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (result.isPresent()) {
|
||||
return result.get();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -41,8 +41,8 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<version>1.19</version>
|
||||
</dependency>
|
||||
<version>1.10</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<resources>
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<groupId>net.sf.json-lib</groupId>
|
||||
<artifactId>json-lib</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<version>2.4.5-talend</version>
|
||||
<version>2.4.2-talend</version>
|
||||
<name>json-lib</name>
|
||||
|
||||
<properties>
|
||||
|
||||
@@ -104,17 +104,16 @@ public final class JSONUtils {
|
||||
return "null";
|
||||
}
|
||||
|
||||
// Shave off trailing zeros. Keep decimal to keep type double
|
||||
// Shave off trailing zeros and decimal point, if possible.
|
||||
|
||||
String s = Double.toString( d );
|
||||
if( s.indexOf( '.' ) > 0 && s.indexOf( 'e' ) < 0 && s.indexOf( 'E' ) < 0 ){
|
||||
while(s.charAt(s.length() - 1) == '0'){
|
||||
if(s.endsWith(".0")){
|
||||
break;
|
||||
}
|
||||
|
||||
while( s.endsWith( "0" ) ){
|
||||
s = s.substring( 0, s.length() - 1 );
|
||||
}
|
||||
}
|
||||
if( s.endsWith( "." ) ){
|
||||
s = s.substring( 0, s.length() - 1 );
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
@@ -441,19 +440,17 @@ public final class JSONUtils {
|
||||
}
|
||||
testValidity( n );
|
||||
|
||||
// Shave off trailing zeros. Keep decimal to keep type double
|
||||
// Shave off trailing zeros and decimal point, if possible.
|
||||
|
||||
String s = n.toString();
|
||||
if( s.indexOf( '.' ) > 0 && s.indexOf( 'e' ) < 0 && s.indexOf( 'E' ) < 0 ){
|
||||
while(s.charAt(s.length() - 1) == '0'){
|
||||
if(s.endsWith(".0")){
|
||||
break;
|
||||
}
|
||||
|
||||
while( s.endsWith( "0" ) ){
|
||||
s = s.substring( 0, s.length() - 1 );
|
||||
}
|
||||
if( s.endsWith( "." ) ){
|
||||
s = s.substring( 0, s.length() - 1 );
|
||||
}
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
|
||||
@@ -163,11 +163,6 @@ public class XMLSerializer {
|
||||
*/
|
||||
private boolean useLongDecimals;
|
||||
|
||||
/**
|
||||
* flag for if parse empty elements as empty strings
|
||||
*/
|
||||
private boolean useEmptyStrings;
|
||||
|
||||
/**
|
||||
* Creates a new XMLSerializer with default options.<br>
|
||||
* <ul>
|
||||
@@ -808,8 +803,6 @@ public class XMLSerializer {
|
||||
clazz = JSONTypes.OBJECT;
|
||||
}else if( JSONTypes.ARRAY.compareToIgnoreCase( clazzText ) == 0 ){
|
||||
clazz = JSONTypes.ARRAY;
|
||||
} else if(JSONTypes.STRING.equalsIgnoreCase(clazzText)) {
|
||||
clazz = JSONTypes.STRING;
|
||||
}
|
||||
}
|
||||
return clazz;
|
||||
@@ -1421,8 +1414,6 @@ public class XMLSerializer {
|
||||
String text = element.getValue();
|
||||
params = StringUtils.split( paramsAttribute.getValue(), "," );
|
||||
setOrAccumulate( jsonObject, key, new JSONFunction( params, text ) );
|
||||
} else if( useEmptyStrings && clazz != null && clazz.equalsIgnoreCase(JSONTypes.STRING) ) {
|
||||
setTextValue(jsonObject, key, element);
|
||||
}else{
|
||||
if( isArray( element, false ) ){
|
||||
setOrAccumulate( jsonObject, key, processArrayElement( element, defaultType ) );
|
||||
@@ -1430,23 +1421,19 @@ public class XMLSerializer {
|
||||
setOrAccumulate( jsonObject, key, simplifyValue( jsonObject,
|
||||
processObjectElement( element, defaultType ) ) );
|
||||
}else{
|
||||
setTextValue(jsonObject, key, element);
|
||||
String value;
|
||||
if( isKeepCData && isCData( element ) ){
|
||||
value = "<![CDATA[" + element.getValue() + "]]>";
|
||||
}else{
|
||||
value = element.getValue();
|
||||
}
|
||||
setOrAccumulate( jsonObject, key, trimSpaceFromValue( value ) );
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void setTextValue(final JSONObject jsonObject, final String key, final Element element) {
|
||||
String value;
|
||||
if( isKeepCData && isCData( element ) ){
|
||||
value = "<![CDATA[" + element.getValue() + "]]>";
|
||||
}else{
|
||||
value = element.getValue();
|
||||
}
|
||||
setOrAccumulate( jsonObject, key, trimSpaceFromValue( value ) );
|
||||
}
|
||||
|
||||
private boolean isCData( Element element ) {
|
||||
if( element.getChildCount() == 1 ){
|
||||
final Node child = element.getChild( 0 );
|
||||
@@ -1506,14 +1493,6 @@ public class XMLSerializer {
|
||||
return str;
|
||||
}
|
||||
|
||||
public void setUseEmptyStrings(boolean useEmptyStrings) {
|
||||
this.useEmptyStrings = useEmptyStrings;
|
||||
}
|
||||
|
||||
public boolean isUseEmptyStrings() {
|
||||
return this.useEmptyStrings;
|
||||
}
|
||||
|
||||
private static class CustomElement extends Element {
|
||||
private static String getName( String name ) {
|
||||
int colon = name.indexOf( ':' );
|
||||
|
||||
@@ -44,25 +44,7 @@ public class TestJSONUtils extends TestCase {
|
||||
}
|
||||
|
||||
public void testDoubleToString_trailingZeros() {
|
||||
assertEquals( "200.0", JSONUtils.doubleToString( 200.00000 ) );
|
||||
}
|
||||
|
||||
public void testDoubleToString() {
|
||||
Map<String, Double> expected = new HashMap<>();
|
||||
expected.put("200.0", 200.0d);
|
||||
expected.put("200.0", 200.000d);
|
||||
expected.put("200.1", 200.1d);
|
||||
expected.put("200.1", 200.10d);
|
||||
expected.put("200.1", 200.1000d);
|
||||
expected.put("200.12345", 200.12345d);
|
||||
expected.put("200.12345", 200.123450000d);
|
||||
expected.put("200.101", 200.101d);
|
||||
expected.put("1.0E-8", 1.0E-8);
|
||||
expected.put("200.0", 200d);
|
||||
|
||||
for(String key : expected.keySet()){
|
||||
assertEquals(key,JSONUtils.doubleToString(expected.get(key)));
|
||||
}
|
||||
assertEquals( "200", JSONUtils.doubleToString( 200.00000 ) );
|
||||
}
|
||||
|
||||
public void testGetFunctionParams() {
|
||||
@@ -119,28 +101,6 @@ public class TestJSONUtils extends TestCase {
|
||||
}
|
||||
}
|
||||
|
||||
public void testNumberToString() {
|
||||
Map<String, Number> expected = new HashMap<>();
|
||||
|
||||
expected.put("0", Integer.valueOf("00000"));
|
||||
expected.put("123", Integer.valueOf("123"));
|
||||
expected.put("-123", Integer.valueOf("-123"));
|
||||
|
||||
expected.put("0.0", Double.valueOf("0"));
|
||||
expected.put("0.0", Double.valueOf("0.00000"));
|
||||
expected.put("120.0001", Double.valueOf("120.0001000"));
|
||||
expected.put("-120.0001", Double.valueOf("-120.0001000"));
|
||||
|
||||
expected.put("0.0", Float.valueOf("0"));
|
||||
expected.put("0.0", Float.valueOf("0.00000"));
|
||||
expected.put("120.0001", Float.valueOf("120.0001000"));
|
||||
expected.put("-120.0001", Float.valueOf("-120.0001000"));
|
||||
|
||||
for(String key : expected.keySet()) {
|
||||
assertEquals(key, JSONUtils.numberToString(expected.get(key)));
|
||||
}
|
||||
}
|
||||
|
||||
public void testQuote_emptyString() {
|
||||
assertEquals( "\"\"", JSONUtils.quote( "" ) );
|
||||
}
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>job-audit</artifactId>
|
||||
<version>1.0.1</version>
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>job-audit</artifactId>
|
||||
<version>1.0</version>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
</properties>
|
||||
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>talend_nexus</id>
|
||||
@@ -23,7 +23,7 @@
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
@@ -46,36 +46,32 @@
|
||||
</releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.talend.daikon</groupId>
|
||||
<artifactId>daikon-audit</artifactId>
|
||||
<version>0.31.8</version>
|
||||
</dependency>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.talend.daikon</groupId>
|
||||
<artifactId>audit-common</artifactId>
|
||||
<version>1.8.0</version>
|
||||
<groupId>org.talend.daikon</groupId>
|
||||
<artifactId>audit-logback</artifactId>
|
||||
<version>0.31.8</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.talend.daikon</groupId>
|
||||
<artifactId>audit-log4j1</artifactId>
|
||||
<version>1.8.0</version>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
<version>1.3.0-alpha4</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-api</artifactId>
|
||||
<version>2.12.1</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
<version>2.12.1</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -18,30 +18,13 @@ public class JobEventAuditLoggerFactory {
|
||||
final AuditConfigurationMap config = AuditConfiguration.loadFromProperties(properties);
|
||||
|
||||
AbstractBackend logger = null;
|
||||
|
||||
//load log4j2 implement firstly
|
||||
String loggerClass = "org.talend.logging.audit.log4j2.Log4j2Backend";
|
||||
String loggerClass = "org.talend.logging.audit.logback.LogbackBackend";
|
||||
try {
|
||||
final Class<?> clz = Class.forName(loggerClass);
|
||||
logger = (AbstractBackend) clz.getConstructor(AuditConfigurationMap.class).newInstance(config);
|
||||
} catch (ReflectiveOperationException e) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
//load log4j1 implement if not found log4j2
|
||||
if (logger == null) {
|
||||
loggerClass = "org.talend.logging.audit.log4j1.Log4j1Backend";
|
||||
try {
|
||||
final Class<?> clz = Class.forName(loggerClass);
|
||||
logger = (AbstractBackend) clz.getConstructor(AuditConfigurationMap.class).newInstance(config);
|
||||
} catch (ReflectiveOperationException e) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
|
||||
if(logger == null) {
|
||||
throw new RuntimeException("Unable to load backend : " + loggerClass);
|
||||
}
|
||||
final Class<?> clz = Class.forName(loggerClass);
|
||||
logger = (AbstractBackend) clz.getConstructor(AuditConfigurationMap.class).newInstance(config);
|
||||
} catch (ReflectiveOperationException e) {
|
||||
throw new RuntimeException("Unable to load backend " + loggerClass, e);
|
||||
}
|
||||
|
||||
final DefaultAuditLoggerBase loggerBase = new DefaultAuditLoggerBase(logger, config);
|
||||
|
||||
@@ -55,4 +38,4 @@ public class JobEventAuditLoggerFactory {
|
||||
return (T) Proxy.newProxyInstance(AuditLoggerFactory.class.getClassLoader(), new Class<?>[] { clz },
|
||||
new ProxyEventAuditLogger(auditLoggerBase));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,8 +8,7 @@ public class JobAuditLoggerTest {
|
||||
|
||||
public static void main(String[] args) {
|
||||
Properties props = new Properties();
|
||||
String root_logger_name = "audit";
|
||||
props.setProperty("root.logger", root_logger_name);
|
||||
props.setProperty("root.logger", "audit");
|
||||
props.setProperty("encoding", "UTF-8");
|
||||
props.setProperty("application.name", "Talend Studio");
|
||||
props.setProperty("service.name", "Talend Studio Job");
|
||||
@@ -20,10 +19,6 @@ public class JobAuditLoggerTest {
|
||||
props.setProperty("appender.file.maxsize", "52428800");
|
||||
props.setProperty("appender.file.maxbackup", "20");
|
||||
props.setProperty("host", "false");
|
||||
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel(root_logger_name, org.apache.logging.log4j.Level.DEBUG);
|
||||
//org.apache.log4j.Logger.getLogger("audit").setLevel(org.apache.log4j.Level.DEBUG);
|
||||
|
||||
final JobAuditLogger logger = JobEventAuditLoggerFactory.createJobAuditLogger(props);
|
||||
Context context = JobContextBuilder.create().jobName("fetch_from_s3_every_day").jobId("jobid_123")
|
||||
.jobVersion("0.1").connectorType("tXMLMAP").connectorId("tXMLMap_1")
|
||||
@@ -36,4 +31,4 @@ public class JobAuditLoggerTest {
|
||||
logger.flowOutput(context);
|
||||
logger.jobstop(context);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talendExcel-1.6-20210127</artifactId>
|
||||
<artifactId>talendExcel-1.4-20190531</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
|
||||
@@ -69,10 +69,6 @@ public class ExcelTool {
|
||||
private int rowAccessWindowSize = SXSSFWorkbook.DEFAULT_WINDOW_SIZE;// used in auto flush
|
||||
|
||||
private boolean isTrackAllColumns = false;
|
||||
|
||||
private boolean isTruncateExceedingCharacters = false;
|
||||
|
||||
private static final int CELL_CHARACTERS_LIMIT = 32767;
|
||||
|
||||
public ExcelTool() {
|
||||
cellStylesMapping = new HashMap<>();
|
||||
@@ -278,10 +274,7 @@ public class ExcelTool {
|
||||
|
||||
public void addCellValue(String stringValue) {
|
||||
addCell();
|
||||
String value = isTruncateExceedingCharacters && stringValue != null && stringValue.length() > CELL_CHARACTERS_LIMIT
|
||||
? stringValue.substring(0, CELL_CHARACTERS_LIMIT)
|
||||
: stringValue;
|
||||
curCell.setCellValue(value);
|
||||
curCell.setCellValue(stringValue);
|
||||
curCell.setCellStyle(getNormalCellStyle());
|
||||
}
|
||||
|
||||
@@ -356,8 +349,4 @@ public class ExcelTool {
|
||||
((SXSSFSheet) sheet).flushRows();
|
||||
}
|
||||
}
|
||||
|
||||
public void setTruncateExceedingCharacters(boolean isTruncateExceedingCharacters) {
|
||||
this.isTruncateExceedingCharacters = isTruncateExceedingCharacters;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talendsap</artifactId>
|
||||
<name>talendsap</name>
|
||||
<version>1.0.2</version>
|
||||
<version>1.0.1</version>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
|
||||
@@ -90,7 +90,6 @@ public class DocumentExtractor {
|
||||
List<Element> tablesAndChangingElements = new ArrayList<Element>(3);
|
||||
tablesAndChangingElements.add(functionElement.element("TABLES"));
|
||||
tablesAndChangingElements.add(functionElement.element("CHANGING"));
|
||||
tablesAndChangingElements.add(functionElement.element("OUTPUT"));
|
||||
|
||||
for(Element tablesOrChangingElement : tablesAndChangingElements) {
|
||||
if (tablesOrChangingElement == null) {
|
||||
@@ -122,10 +121,6 @@ public class DocumentExtractor {
|
||||
}
|
||||
result.add(row);
|
||||
}
|
||||
|
||||
if(!result.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
@@ -49,11 +49,11 @@ public class DocumentHelper {
|
||||
}
|
||||
}
|
||||
|
||||
public void addSingleParameter(String name, String value, SAPParameterType parameter_type) {
|
||||
public void addSingleParameter(String name, String value, boolean isChanging) {
|
||||
if(value == null) {
|
||||
value = "";
|
||||
}
|
||||
if (parameter_type == SAPParameterType.CHANGING) {
|
||||
if (isChanging) {
|
||||
correctChanging();
|
||||
changing.addElement(name).setText(value);
|
||||
} else {
|
||||
@@ -62,8 +62,8 @@ public class DocumentHelper {
|
||||
}
|
||||
}
|
||||
|
||||
public void addStructParameter(String name, SAPParameterType parameter_type) {
|
||||
if (parameter_type == SAPParameterType.CHANGING) {
|
||||
public void addStructParameter(String name, boolean isChanging) {
|
||||
if (isChanging) {
|
||||
correctChanging();
|
||||
currentStruct = changing.addElement(name);
|
||||
} else {
|
||||
@@ -79,16 +79,13 @@ public class DocumentHelper {
|
||||
currentStruct.addElement(name).setText(value);
|
||||
}
|
||||
|
||||
public void addTableParameter(String name, SAPParameterType parameter_type) {
|
||||
if (parameter_type == SAPParameterType.CHANGING) {
|
||||
public void addTableParameter(String name, boolean isChanging) {
|
||||
if (isChanging) {
|
||||
correctChanging();
|
||||
currentTable = changing.addElement(name);
|
||||
} else if(parameter_type == SAPParameterType.TABLES) {
|
||||
} else {
|
||||
correctTables();
|
||||
currentTable = tables.addElement(name);
|
||||
} else {
|
||||
correctInput();
|
||||
currentTable = input.addElement(name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,18 +108,18 @@ public class DocumentHelper {
|
||||
DocumentHelper helper = new DocumentHelper();
|
||||
helper.setFunctionName("READ_TABLE_FUNCTION");
|
||||
|
||||
helper.addSingleParameter("ID", "1", SAPParameterType.CHANGING);
|
||||
helper.addSingleParameter("NAME", "gaoyan", SAPParameterType.IMPORT);
|
||||
helper.addSingleParameter("ID", "1", true);
|
||||
helper.addSingleParameter("NAME", "gaoyan", false);
|
||||
|
||||
helper.addStructParameter("INFO", SAPParameterType.CHANGING);
|
||||
helper.addStructParameter("INFO", true);
|
||||
helper.addStructChildParameter("ID", "2");
|
||||
helper.addStructChildParameter("NAME", "wangwei");
|
||||
|
||||
helper.addStructParameter("INFO1", SAPParameterType.IMPORT);
|
||||
helper.addStructParameter("INFO1", false);
|
||||
helper.addStructChildParameter("ID1", "4");
|
||||
helper.addStructChildParameter("NAME1", "momo");
|
||||
|
||||
helper.addTableParameter("TABLE1", SAPParameterType.TABLES);
|
||||
helper.addTableParameter("TABLE1", false);
|
||||
for (int i = 0; i < 200000; i++) {
|
||||
helper.addTableRow();
|
||||
helper.addTableRowChildParameter("c1", i + "");
|
||||
@@ -135,7 +132,7 @@ public class DocumentHelper {
|
||||
helper.addTableRowChildParameter("c8", "wangwei" + i);
|
||||
}
|
||||
|
||||
helper.addTableParameter("TABLE2", SAPParameterType.TABLES);
|
||||
helper.addTableParameter("TABLE2", false);
|
||||
for (int i = 0; i < 2; i++) {
|
||||
helper.addTableRow();
|
||||
helper.addTableRowChildParameter("ID4", i + "");
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
package org.talend.sap;
|
||||
|
||||
public enum SAPParameterType {
|
||||
IMPORT,
|
||||
CHANGING,
|
||||
TABLES,
|
||||
EXPORT
|
||||
}
|
||||
@@ -46,7 +46,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<version>1.19</version>
|
||||
<version>1.10</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
@@ -55,4 +55,4 @@
|
||||
<version>6.0.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
</project>
|
||||
@@ -5,9 +5,9 @@
|
||||
>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>thashfile</artifactId>
|
||||
<version>3.1-20190910</version>
|
||||
<version>3.0-20170711</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>thashfile</name>
|
||||
@@ -45,8 +45,8 @@
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.5.1</version>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
<source>1.7</source>
|
||||
<target>1.7</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
|
||||
@@ -9,44 +9,45 @@ import java.util.concurrent.ConcurrentHashMap;
|
||||
import org.talend.designer.components.hashfile.memory.AdvancedMemoryHashFile;
|
||||
|
||||
public class MapHashFile {
|
||||
// use this map instead of globalMap
|
||||
//use this map instead of globalMap
|
||||
private Map<String, AdvancedMemoryHashFile> resourceMap = new ConcurrentHashMap<>();
|
||||
// keep the present key of AdvancedMemoryHashFile as key and the previous key as
|
||||
// value
|
||||
//keep the present key of AdvancedMemoryHashFile as key and the previous key as value
|
||||
private Map<String, String> keyMap = new ConcurrentHashMap<>();
|
||||
// singleton
|
||||
//singleton
|
||||
private static final MapHashFile mhf = new MapHashFile();
|
||||
|
||||
public static TalendMultiThreadLockMap resourceLockMap = new TalendMultiThreadLockMap();
|
||||
public static TalendMultiThreadLockMap resourceLockMap = new TalendMultiThreadLockMap();
|
||||
|
||||
public static class TalendMultiThreadLockMap {
|
||||
public static class TalendMultiThreadLockMap {
|
||||
|
||||
private Map<Object, Object> tMultiTheadLockMap = new ConcurrentHashMap<>();
|
||||
|
||||
public Object get(Object key) {
|
||||
return tMultiTheadLockMap.computeIfAbsent(key, k -> new Object());
|
||||
}
|
||||
|
||||
public void remove(Object key) {
|
||||
tMultiTheadLockMap.remove(key);
|
||||
}
|
||||
}
|
||||
private Map<Object, Object> tMultiTheadLockMap = new HashMap<Object, Object>();
|
||||
|
||||
public synchronized Object get(Object key) {
|
||||
if (tMultiTheadLockMap.get(key) == null) {
|
||||
tMultiTheadLockMap.put(key, new Object());
|
||||
}
|
||||
return tMultiTheadLockMap.get(key);
|
||||
}
|
||||
|
||||
public synchronized void remove(Object key){
|
||||
tMultiTheadLockMap.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
private MapHashFile() {
|
||||
}
|
||||
|
||||
public static MapHashFile getMapHashFile() {
|
||||
return mhf;
|
||||
}
|
||||
|
||||
// get the linked AdvancedMemoryHashFile
|
||||
|
||||
//get the linked AdvancedMemoryHashFile
|
||||
public AdvancedMemoryHashFile getAdvancedMemoryHashFile(String key) {
|
||||
AdvancedMemoryHashFile amhf = resourceMap.get(key);
|
||||
String prekey = keyMap.get(key);
|
||||
// if present AdvancedMemoryHashFile is null get the AdvancedMemoryHashFile
|
||||
// before present.
|
||||
//if present AdvancedMemoryHashFile is null get the AdvancedMemoryHashFile before present.
|
||||
int size = keyMap.size();
|
||||
while (amhf == null && (size--) > 0) {
|
||||
while(amhf==null && (size--)>0){
|
||||
amhf = resourceMap.get(prekey);
|
||||
prekey = keyMap.get(prekey);
|
||||
}
|
||||
@@ -60,18 +61,16 @@ public class MapHashFile {
|
||||
public Map<String, String> getKeyMap() {
|
||||
return keyMap;
|
||||
}
|
||||
|
||||
public void clearCache(String key) {
|
||||
public void clearCache(String key){
|
||||
clearChildCache(getRootCache(key));
|
||||
}
|
||||
|
||||
public void clearChildCache(String root) {
|
||||
public void clearChildCache(String root){
|
||||
Set<String> set = keyMap.keySet();
|
||||
synchronized (keyMap) {
|
||||
synchronized(keyMap) {
|
||||
Iterator<String> it = set.iterator();
|
||||
while (it.hasNext()) {
|
||||
while(it.hasNext()){
|
||||
String key = it.next();
|
||||
if (root.equals(keyMap.get(key))) {
|
||||
if(root.equals(keyMap.get(key))){
|
||||
this.resourceMap.remove(key);
|
||||
this.keyMap.remove(key);
|
||||
clearChildCache(key);
|
||||
@@ -80,11 +79,11 @@ public class MapHashFile {
|
||||
}
|
||||
this.resourceMap.remove(root);
|
||||
}
|
||||
|
||||
public String getRootCache(String cache) {
|
||||
|
||||
public String getRootCache(String cache){
|
||||
String root;
|
||||
while ((root = keyMap.get(cache)) != null) {
|
||||
cache = root;
|
||||
while((root = keyMap.get(cache))!=null){
|
||||
cache=root;
|
||||
}
|
||||
return cache;
|
||||
}
|
||||
|
||||
@@ -54,11 +54,11 @@ for(IConnection conn : outgoingConns) {
|
||||
<%
|
||||
}
|
||||
}
|
||||
log4jCodeGenerateUtil.query(node);
|
||||
%>
|
||||
query_<%=cid %> = <%=dbquery%>;
|
||||
whetherReject_<%=cid%> = false;
|
||||
<%
|
||||
log4jCodeGenerateUtil.query(node, "query_" + cid);
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
if ((metadatas!=null)&&(metadatas.size()>0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
@@ -131,7 +131,7 @@ try {
|
||||
<%
|
||||
}
|
||||
}
|
||||
log4jCodeGenerateUtil.logInfo(node,"info",cid+" - Execute the query: '\" + query_" + cid + " + \"' has finished.");
|
||||
log4jCodeGenerateUtil.logInfo(node,"info",cid+" - Execute the query: '\" + "+dbquery +" + \"' has finished.");
|
||||
%>
|
||||
<% //feature 0010425
|
||||
if(usePrepareStatement){
|
||||
|
||||
@@ -24,9 +24,6 @@
|
||||
String user = ElementParameterParser.getValue(node, "__USER__");
|
||||
String node_type = ElementParameterParser.getValue(node, "__NODE_TYPE__");
|
||||
String node_count = ElementParameterParser.getValue(node, "__NODE_COUNT__");
|
||||
if (node_count.isEmpty()) {
|
||||
node_count = "1";
|
||||
}
|
||||
|
||||
boolean isLog4jEnabled = "true".equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
|
||||
@@ -91,46 +88,52 @@
|
||||
.withMasterUsername(<%=user%>)
|
||||
.withMasterUserPassword(decryptedPwd_<%=cid%>)
|
||||
|
||||
.withNodeType(<%=node_type%>);
|
||||
.withNodeType(<%=node_type%>)
|
||||
|
||||
if (<%=node_count%> > 1) {
|
||||
request_<%=cid%> = request_<%=cid%>.withNumberOfNodes(<%=node_count%>);
|
||||
} else {
|
||||
request_<%=cid%> = request_<%=cid%>.withClusterType("single-node");
|
||||
}
|
||||
|
||||
//advanced settings
|
||||
<%if(parameterUtil.isValid(parameter_group_name)) {%>
|
||||
request_<%=cid%> = request_<%=cid%>.withClusterParameterGroupName(<%=parameter_group_name%>);
|
||||
<%}%>
|
||||
|
||||
<%if(parameterUtil.isValid(subnet_group_name)) {%>
|
||||
request_<%=cid%> = request_<%=cid%>.withClusterSubnetGroupName(<%=subnet_group_name%>);
|
||||
<%}%>
|
||||
|
||||
<%
|
||||
if(publicly_accessible) {
|
||||
%>
|
||||
request_<%=cid%> = request_<%=cid%>.withPubliclyAccessible(true);
|
||||
<%if(set_public_ip_address) {%>
|
||||
request_<%=cid%> = request_<%=cid%>.withElasticIp(<%=elastic_ip%>);
|
||||
<%
|
||||
if(new Integer(node_count) > 1){
|
||||
%>
|
||||
.withNumberOfNodes(<%=node_count%>)
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
.withClusterType("single-node")
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
//advanced settings
|
||||
<%if(parameterUtil.isValid(parameter_group_name)) {%>
|
||||
.withClusterParameterGroupName(<%=parameter_group_name%>)
|
||||
<%}%>
|
||||
|
||||
<%if(parameterUtil.isValid(subnet_group_name)) {%>
|
||||
.withClusterSubnetGroupName(<%=subnet_group_name%>)
|
||||
<%}%>
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
request_<%=cid%> = request_<%=cid%>.withPubliclyAccessible(false);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
<%if(parameterUtil.isValid(availability_zone)) {%>
|
||||
request_<%=cid%> = request_<%=cid%>.withAvailabilityZone(<%=availability_zone%>);
|
||||
<%}%>
|
||||
|
||||
<%if(parameterUtil.isValid(vpc_security_groupids)) {%>
|
||||
request_<%=cid%> = request_<%=cid%>.withVpcSecurityGroupIds(<%=vpc_security_groupids%>.split(","));
|
||||
<%}%>
|
||||
|
||||
|
||||
<%
|
||||
if(publicly_accessible) {
|
||||
%>
|
||||
.withPubliclyAccessible(true)
|
||||
<%if(set_public_ip_address) {%>
|
||||
.withElasticIp(<%=elastic_ip%>)
|
||||
<%}%>
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
.withPubliclyAccessible(false)
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
<%if(parameterUtil.isValid(availability_zone)) {%>
|
||||
.withAvailabilityZone(<%=availability_zone%>)
|
||||
<%}%>
|
||||
|
||||
<%if(parameterUtil.isValid(vpc_security_groupids)) {%>
|
||||
.withVpcSecurityGroupIds(<%=vpc_security_groupids%>.split(","))
|
||||
<%}%>
|
||||
;
|
||||
|
||||
com.amazonaws.services.redshift.model.Cluster result_<%=cid%> = client_<%=cid%>.createCluster(request_<%=cid%>);
|
||||
<%if(isLog4jEnabled) {%>
|
||||
|
||||
@@ -149,7 +149,7 @@
|
||||
<IMPORTS>
|
||||
<IMPORT
|
||||
NAME="Talen File Enhanced"
|
||||
MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar"
|
||||
MODULE="talend_file_enhanced_20070724.jar" MVN="mvn:org.talend.libraries/talend_file_enhanced_20070724/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced_20070724.jar"
|
||||
REQUIRED="true"
|
||||
/>
|
||||
</IMPORTS>
|
||||
|
||||
@@ -32,8 +32,6 @@
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
|
||||
boolean useCustomNullMarker = ElementParameterParser.getBooleanValue(node, "__USE_CUSTOM_NULL_MARKER__");
|
||||
String nullMarker = useCustomNullMarker ? ElementParameterParser.getValue(node, "__CUSTOM_NULL_MARKER__") : "\"\\\\N\"";
|
||||
String passwordFieldName = "";
|
||||
|
||||
if (authMode.equals("OAUTH")) {
|
||||
@@ -362,7 +360,7 @@
|
||||
queryLoad_<%=cid%>.setDestinationTable(destinationTable_<%=cid%>);
|
||||
queryLoad_<%=cid%>.setSourceUris(java.util.Arrays.asList(<%=ElementParameterParser.getValue(node, "__GS_FILE__")%>));
|
||||
queryLoad_<%=cid%>.setSkipLeadingRows(<%=ElementParameterParser.getValue(node, "__GS_FILE_HEADER__")%>);
|
||||
queryLoad_<%=cid%>.setNullMarker(<%= nullMarker %>);
|
||||
queryLoad_<%=cid%>.setNullMarker("\\N");
|
||||
config_<%=cid%>.setLoad(queryLoad_<%=cid%>);
|
||||
|
||||
job_<%=cid%>.setConfiguration(config_<%=cid%>);
|
||||
@@ -538,6 +536,7 @@
|
||||
|
||||
com.google.cloud.bigquery.Schema schema_<%=cid%> = com.google.cloud.bigquery.Schema.of(fields_<%=cid%>);
|
||||
com.google.cloud.bigquery.TableInfo tableInfo_<%=cid%> = com.google.cloud.bigquery.TableInfo.newBuilder(tableId_<%=cid%>, com.google.cloud.bigquery.StandardTableDefinition.of(schema_<%=cid%>)).build();
|
||||
table_<%=cid%> = bigquery_<%=cid%>.create(tableInfo_<%=cid%>);
|
||||
loadJobBuilder_<%=cid%>.setSchema(schema_<%=cid%>);
|
||||
}
|
||||
|
||||
@@ -574,16 +573,15 @@
|
||||
loadJobBuilder_<%=cid%>.setDestinationTable(tableId_<%=cid%>);
|
||||
com.google.cloud.bigquery.CsvOptions.Builder csvOptions_<%=cid%> = com.google.cloud.bigquery.CsvOptions.newBuilder();
|
||||
csvOptions_<%=cid%>.setAllowQuotedNewLines(true);
|
||||
csvOptions_<%=cid%>.setSkipLeadingRows(<%=ElementParameterParser.getValue(node, "__GS_FILE_HEADER__")%>);
|
||||
|
||||
<%if("true".equals(ElementParameterParser.getValue(node, "__SET_FIELD_DELIMITER__"))) {
|
||||
%>
|
||||
csvOptions_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>);
|
||||
loadJobBuilder_<%=cid%>.setFormatOptions(csvOptions_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>).build());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setFormatOptions(csvOptions_<%=cid%>.build());
|
||||
loadJobBuilder_<%=cid%>.setNullMarker(<%= nullMarker %>);
|
||||
|
||||
loadJobBuilder_<%=cid%>.setNullMarker("\\N");
|
||||
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.of(loadJobBuilder_<%=cid%>.build()));
|
||||
job_<%=cid%> = job_<%=cid%>.waitFor(com.google.cloud.RetryOption.initialRetryDelay(org.threeten.bp.Duration.ofSeconds(1)));
|
||||
if (job_<%=cid%> != null && job_<%=cid%>.getStatus().getError() == null) {
|
||||
|
||||
@@ -173,16 +173,10 @@
|
||||
SHOW_IF="(SET_FIELD_DELIMITER == 'true')">
|
||||
<DEFAULT>","</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="USE_CUSTOM_NULL_MARKER" FIELD="CHECK" NUM_ROW="25">
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="CUSTOM_NULL_MARKER" FIELD="TEXT" NUM_ROW="25" SHOW_IF="USE_CUSTOM_NULL_MARKER=='true'">
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="30" SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'">
|
||||
<PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="25" SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'">
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="ENCODING" FIELD="ENCODING_TYPE" NUM_ROW="35"
|
||||
<PARAMETER NAME="ENCODING" FIELD="ENCODING_TYPE" NUM_ROW="30"
|
||||
REQUIRED="true" REPOSITORY_VALUE="ENCODING">
|
||||
<DEFAULT>"ISO-8859-15"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
@@ -40,8 +40,6 @@ ENCODING.NAME=Encoding
|
||||
TOKEN_NAME.NAME=token properties File Name
|
||||
SET_FIELD_DELIMITER.NAME=Set the field delimiter
|
||||
FIELD_DELIMITER.NAME=
|
||||
USE_CUSTOM_NULL_MARKER.NAME=Use custom null marker
|
||||
CUSTOM_NULL_MARKER.NAME=
|
||||
DROP.NAME=Drop table if exists
|
||||
|
||||
DIE_ON_ERROR.NAME=Die on error
|
||||
|
||||
@@ -21,7 +21,7 @@ class BigQueryUtil_<%=cid%> {
|
||||
private String genTempName(String prefix){
|
||||
return "temp_" + prefix + java.util.UUID.randomUUID().toString().replaceAll("-", "") + "<%=cid%>".toLowerCase().replaceAll("[^a-z0-9]", "0").replaceAll("^[^a-z]", "a") + Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
|
||||
}
|
||||
|
||||
|
||||
public void cleanup() throws Exception{
|
||||
if(useLargeResult){
|
||||
bigqueryclient.tables().delete(projectId, tempDataset, tempTable).execute();
|
||||
@@ -38,11 +38,11 @@ class BigQueryUtil_<%=cid%> {
|
||||
job.setConfiguration(config);
|
||||
List<com.google.api.services.bigquery.model.TableReference> referencedTables = bigqueryclient.jobs().insert(projectId, job).execute().getStatistics().getQuery().getReferencedTables();
|
||||
if(referencedTables != null && !referencedTables.isEmpty()) {
|
||||
location = bigqueryclient.tables().get(referencedTables.get(0).getProjectId(), referencedTables.get(0).getDatasetId(), referencedTables.get(0).getTableId()).execute().getLocation();
|
||||
location = bigqueryclient.tables().get(projectId, referencedTables.get(0).getDatasetId(), referencedTables.get(0).getTableId()).execute().getLocation();
|
||||
}
|
||||
return location;
|
||||
}
|
||||
|
||||
|
||||
private void createDataset(String location) throws Exception {
|
||||
com.google.api.services.bigquery.model.Dataset dataset = new com.google.api.services.bigquery.model.Dataset().setDatasetReference(new com.google.api.services.bigquery.model.DatasetReference().setProjectId(projectId).setDatasetId(tempDataset));
|
||||
if(location != null) {
|
||||
@@ -53,11 +53,11 @@ class BigQueryUtil_<%=cid%> {
|
||||
dataset.setDescription(description);
|
||||
bigqueryclient.datasets().insert(projectId, dataset).execute();
|
||||
}
|
||||
|
||||
|
||||
public com.google.api.services.bigquery.model.Job executeQuery(String query, boolean useLargeResult) throws Exception{
|
||||
return executeQuery(query, useLargeResult, true);
|
||||
}
|
||||
|
||||
|
||||
public com.google.api.services.bigquery.model.Job executeQuery(String query, boolean useLargeResult, boolean useLegacySql) throws Exception{
|
||||
com.google.api.services.bigquery.model.JobConfigurationQuery queryConfig = new com.google.api.services.bigquery.model.JobConfigurationQuery();
|
||||
queryConfig.setQuery(query);
|
||||
@@ -74,13 +74,13 @@ class BigQueryUtil_<%=cid%> {
|
||||
.setDatasetId(tempDataset)
|
||||
.setTableId(tempTable));
|
||||
}
|
||||
|
||||
|
||||
com.google.api.services.bigquery.model.JobConfiguration config = new com.google.api.services.bigquery.model.JobConfiguration();
|
||||
config.setQuery(queryConfig);
|
||||
|
||||
|
||||
com.google.api.services.bigquery.model.Job job = new com.google.api.services.bigquery.model.Job();
|
||||
job.setConfiguration(config);
|
||||
|
||||
|
||||
com.google.api.services.bigquery.model.Job insert = null;
|
||||
com.google.api.services.bigquery.model.JobReference jobId = null;
|
||||
try {
|
||||
@@ -122,7 +122,7 @@ class BigQueryUtil_<%=cid%> {
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
@@ -146,7 +146,7 @@ class BigQueryUtil_<%=cid%> {
|
||||
<%
|
||||
}
|
||||
%>
|
||||
return executeQuery(query, true, useLegacySql);
|
||||
return executeQuery(query, true);
|
||||
}
|
||||
<%}%>
|
||||
// Do not throw exception to avoid behavior changed(because it may throw "duplicate" exception which do not throw before);
|
||||
@@ -164,13 +164,13 @@ class BigQueryUtil_<%=cid%> {
|
||||
}// else job successful
|
||||
break;
|
||||
}
|
||||
// Pause execution for one second before polling job status again, to
|
||||
// Pause execution for one second before polling job status again, to
|
||||
// reduce unnecessary calls to the BigQUery API and lower overall
|
||||
// application bandwidth.
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
|
||||
|
||||
return insert;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -221,9 +221,10 @@
|
||||
}
|
||||
|
||||
String query_<%=cid%> = <%=query%>;
|
||||
<% if(isLog4jEnabled) { %>
|
||||
log.info("<%=cid%> - query " + <%=query%>);
|
||||
<% } %>
|
||||
String dataset = query_<%=cid%>.substring(query_<%=cid%>.indexOf("from") + 4, query_<%=cid%>.indexOf(".")).trim();
|
||||
String temp_table = "temp_" + dataset + java.util.UUID.randomUUID().toString().replaceAll("-", "")
|
||||
+ "<%=cid%>".toLowerCase().replaceAll("[^a-z0-9]", "0").replaceAll("^[^a-z]", "a")
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
|
||||
|
||||
com.google.cloud.bigquery.BigQuery bigquery_<%=cid%> = com.google.cloud.bigquery.BigQueryOptions.newBuilder()
|
||||
.setCredentials(credentials_<%=cid%>)
|
||||
@@ -232,55 +233,25 @@
|
||||
.getService();
|
||||
|
||||
com.google.cloud.bigquery.QueryJobConfiguration.Builder queryConfiguration_<%=cid%> = com.google.cloud.bigquery.QueryJobConfiguration.newBuilder(<%=query%>).setUseLegacySql(<%=useLegacySql%>);
|
||||
|
||||
<%
|
||||
if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) {
|
||||
%>
|
||||
queryConfiguration_<%=cid%>.setDestinationTable(com.google.cloud.bigquery.TableId.of(dataset, temp_table)).setAllowLargeResults(true);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId.of(java.util.UUID.randomUUID().toString());
|
||||
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.newBuilder(queryConfiguration_<%=cid%>.build()).setJobId(jobId_<%=cid%>).build());
|
||||
|
||||
<%if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) {%>
|
||||
|
||||
com.google.cloud.bigquery.QueryJobConfiguration jobConfDryRun_<%=cid%> = queryConfiguration_<%=cid%> .setDryRun(true).build();
|
||||
com.google.cloud.bigquery.Job jobDryRun_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.of(jobConfDryRun_<%=cid%>));
|
||||
|
||||
String queryLocation_<%=cid%> =jobDryRun_<%=cid%>.getJobId().getLocation();
|
||||
String location_<%=cid%> = queryLocation_<%=cid%> == null ? "US" : queryLocation_<%=cid%>;
|
||||
String tempDataset_<%=cid%> = java.util.UUID.randomUUID().toString().replaceAll("-", "")
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt())
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
|
||||
String tempTable_<%=cid%> = java.util.UUID.randomUUID().toString().replaceAll("-", "")
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt())
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
|
||||
<% if(isLog4jEnabled){ %>
|
||||
log.info("<%=cid%> - query location :" + queryLocation_<%=cid%>);
|
||||
log.info("<%=cid%> - temporary dataset location :" + location_<%=cid%>);
|
||||
log.info("<%=cid%> - temporary Dataset name : " + tempDataset_<%=cid%>);
|
||||
log.info("<%=cid%> - temporary table name : " + tempTable_<%=cid%>);
|
||||
<% } %>
|
||||
|
||||
com.google.cloud.bigquery.DatasetInfo datasetInfo_<%=cid%> = com.google.cloud.bigquery.DatasetInfo.newBuilder(tempDataset_<%=cid%>).setLocation(location_<%=cid%>).build();
|
||||
com.google.cloud.bigquery.Dataset dataset_<%=cid%> = bigquery_<%=cid%> .create(datasetInfo_<%=cid%>);
|
||||
|
||||
queryConfiguration_<%=cid%>
|
||||
.setDryRun(false)
|
||||
.setAllowLargeResults(true)
|
||||
.setDestinationTable(com.google.cloud.bigquery.TableId.of(tempDataset_<%=cid%>, tempTable_<%=cid%>));
|
||||
|
||||
com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId
|
||||
.newBuilder().setProject(<%=projectId%>)
|
||||
.setJob(java.util.UUID.randomUUID().toString())
|
||||
.setLocation(location_<%=cid%>)
|
||||
.build();
|
||||
|
||||
<% if(isLog4jEnabled){ %>
|
||||
log.info("<%=cid%> - job location : " + jobId_<%=cid%>.getLocation());
|
||||
<% } %>
|
||||
<% } else { %>
|
||||
com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId.of(<%=projectId%>,java.util.UUID.randomUUID().toString());
|
||||
|
||||
<% } %>
|
||||
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.newBuilder(queryConfiguration_<%=cid%>.build()).setJobId(jobId_<%=cid%>).build());
|
||||
|
||||
|
||||
<% if(isLog4jEnabled) { %>
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Sending job " + jobId_<%=cid%> + " with query: " + <%=query%>);
|
||||
<% } %>
|
||||
|
||||
<%
|
||||
}
|
||||
%>
|
||||
job_<%=cid%> = job_<%=cid%>.waitFor();
|
||||
|
||||
if (job_<%=cid%> == null) {
|
||||
@@ -289,9 +260,13 @@
|
||||
throw new RuntimeException(job_<%=cid%>.getStatus().getError().toString());
|
||||
}
|
||||
|
||||
<% if(isLog4jEnabled) { %>
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Job " + jobId_<%=cid%> + " finished successfully.");
|
||||
<% } %>
|
||||
<%
|
||||
}
|
||||
%>
|
||||
com.google.cloud.bigquery.TableResult result_<%=cid%> = job_<%=cid%>.getQueryResults();
|
||||
long nb_line_<%=cid%> = 0;
|
||||
<%
|
||||
|
||||
@@ -10,10 +10,7 @@ imports="
|
||||
CodeGeneratorArgument codeGenArgument = (CodeGeneratorArgument) argument;
|
||||
INode node = (INode)codeGenArgument.getArgument();
|
||||
String cid = node.getUniqueName();
|
||||
String authMode = ElementParameterParser.getValue(node,"__AUTH_MODE__");
|
||||
String projectId = ElementParameterParser.getValue(node,"__PROJECT_ID__");
|
||||
String resultSizeType = ElementParameterParser.getValue(node,"__RESULT_SIZE__");
|
||||
|
||||
String authMode = ElementParameterParser.getValue(node,"__AUTH_MODE__");
|
||||
if (authMode.equals("OAUTH")) {
|
||||
%>
|
||||
}
|
||||
@@ -27,10 +24,7 @@ imports="
|
||||
} else if (authMode.equals("SERVICEACCOUNT")) {
|
||||
%>
|
||||
}
|
||||
<% if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) { %>
|
||||
com.google.cloud.bigquery.DatasetId datasetId_<%=cid%> = com.google.cloud.bigquery.DatasetId.of(<%=projectId%>, tempDataset_<%=cid%>);
|
||||
bigquery_<%=cid%>.delete(datasetId_<%=cid%>, com.google.cloud.bigquery.BigQuery.DatasetDeleteOption.deleteContents());
|
||||
<% }%>
|
||||
bigquery_<%=cid%>.delete(com.google.cloud.bigquery.TableId.of(dataset, temp_table));
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
|
||||
@@ -55,12 +55,7 @@
|
||||
<PARAMETER NAME="TEXT_ENCLOSURE" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" MAX_LENGTH="1" REPOSITORY_VALUE="TEXT_ENCLOSURE" SHOW="false">
|
||||
<DEFAULT>"\""</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="USE_CUSTOM_NULL_MARKER" FIELD="CHECK" NUM_ROW="35">
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="CUSTOM_NULL_MARKER" FIELD="TEXT" NUM_ROW="35" SHOW_IF="USE_CUSTOM_NULL_MARKER=='true'">
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="CREATE" FIELD="CHECK" REQUIRED="true" NUM_ROW="40">
|
||||
<DEFAULT>true</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
@@ -25,8 +25,7 @@
|
||||
boolean checkDiskSpace = ("true").equals(ElementParameterParser.getValue(node,"__CHECK_DISK_SPACE__"));
|
||||
boolean flushOnRow = ("true").equals(ElementParameterParser.getValue(node, "__FLUSHONROW__"));
|
||||
String flushMod = ElementParameterParser.getValue(node, "__FLUSHONROW_NUM__");
|
||||
boolean useCustomNullMarker = ElementParameterParser.getBooleanValue(node, "__USE_CUSTOM_NULL_MARKER__");
|
||||
String nullMarker = useCustomNullMarker ? ElementParameterParser.getValue(node, "__CUSTOM_NULL_MARKER__") : "\"\\\\N\"";
|
||||
|
||||
String encoding = ElementParameterParser.getValue(node,"__ENCODING__");
|
||||
|
||||
List< ? extends IConnection> conns = node.getIncomingConnections();
|
||||
@@ -36,8 +35,8 @@
|
||||
if (conn.getLineStyle().hasConnectionCategory(IConnectionCategory.DATA)) {
|
||||
List<IMetadataColumn> columns = metadata.getListColumns();
|
||||
int sizeColumns = columns.size();
|
||||
%>
|
||||
String[] row_<%=cid%>=new String[]{<%for(int j=0;j<sizeColumns;j++){%> <%= nullMarker %>,<%}%>}; //empty value must be defined by null marker('\N' is default in bulk file)
|
||||
%>
|
||||
String[] row_<%=cid%>=new String[]{<%for(int j=0;j<sizeColumns;j++){%>"\\N",<%}%>}; //empty value must be NULL('\N' in bulk file)
|
||||
<%
|
||||
for (int i = 0; i < sizeColumns; i++) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
@@ -65,10 +64,24 @@
|
||||
%>
|
||||
row_<%=cid%>[<%=i%>] = <%=conn.getName() %>.<%=column.getLabel() %>;
|
||||
<%
|
||||
}else if(javaType == JavaTypesManager.DATE){
|
||||
%>
|
||||
row_<%=cid%>[<%=i%>] = FormatterUtils.format_Date(<%=conn.getName() %>.<%=column.getLabel() %>, <%=(pattern != null && !("\"\"").equals(pattern)) ? pattern : "\"yyyy-MM-dd\""%>);
|
||||
<%
|
||||
}else if(javaType == JavaTypesManager.DATE && pattern == null){
|
||||
%>
|
||||
row_<%=cid%>[<%=i%>] = FormatterUtils.format_Date(<%=conn.getName() %>.<%=column.getLabel() %>, "yyyy-MM-dd");
|
||||
<%
|
||||
}else if(javaType == JavaTypesManager.DATE && pattern != null){
|
||||
if(pattern.length() > 12){
|
||||
%>
|
||||
row_<%=cid%>[<%=i%>] = FormatterUtils.format_Date(<%=conn.getName() %>.<%=column.getLabel() %>, "yyyy-MM-dd HH:mm:ss");
|
||||
<%
|
||||
}else if(pattern.length() == 12 || "\"\"".equals(pattern)) {
|
||||
%>
|
||||
row_<%=cid%>[<%=i%>] = FormatterUtils.format_Date(<%=conn.getName() %>.<%=column.getLabel() %>, "yyyy-MM-dd");
|
||||
<%
|
||||
}else {
|
||||
%>
|
||||
row_<%=cid%>[<%=i%>] = FormatterUtils.format_Date(<%=conn.getName() %>.<%=column.getLabel() %>, <%=pattern%>);
|
||||
<%
|
||||
}
|
||||
}else if(javaType == JavaTypesManager.BYTE_ARRAY){
|
||||
%>
|
||||
row_<%=cid%>[<%=i%>] = java.nio.charset.Charset.forName(<%=encoding %>).decode(java.nio.ByteBuffer.wrap(<%=conn.getName() %>.<%=column.getLabel() %>)).toString();
|
||||
@@ -87,7 +100,7 @@
|
||||
}
|
||||
%>
|
||||
} else {
|
||||
row_<%=cid%>[<%=i%>] = <%= nullMarker %>;
|
||||
row_<%=cid%>[<%=i%>] = "\\N";
|
||||
}
|
||||
<%
|
||||
}
|
||||
|
||||
@@ -24,5 +24,4 @@ RECORDS_CONTAIN_NULL_VALUE.NAME=Records contain NULL value
|
||||
PROPERTY.NAME=Property Type
|
||||
CHECK_DISK_SPACE.NAME=Check disk space
|
||||
|
||||
USE_CUSTOM_NULL_MARKER.NAME=Use custom null marker
|
||||
CUSTOM_NULL_MARKER.NAME=
|
||||
|
||||
|
||||
@@ -17,40 +17,40 @@ imports="
|
||||
CodeGeneratorArgument codeGenArgument = (CodeGeneratorArgument) argument;
|
||||
INode node = (INode)codeGenArgument.getArgument();
|
||||
String cid = node.getUniqueName();
|
||||
|
||||
|
||||
String useExistingConn = ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__");
|
||||
String accessToken = ElementParameterParser.getValue(node, "__ACCESS_TOKEN__");
|
||||
String refreshToken = ElementParameterParser.getValue(node, "__REFRESH_TOKEN__");
|
||||
|
||||
|
||||
String clientId = ElementParameterParser.getValue(node, "__CLIENT_ID__");
|
||||
String clientSecret = ElementParameterParser.getValue(node, "__CLIENT_SECRET__");
|
||||
|
||||
|
||||
String connection = ElementParameterParser.getValue(node,"__CONNECTION__");
|
||||
String connectionKey = "\"conn_" + connection+"\"";
|
||||
|
||||
|
||||
String path = ElementParameterParser.getValue(node, "__PATH__");
|
||||
String filelistType = ElementParameterParser.getValue(node, "__LIST_MODE__");
|
||||
boolean includeSubDirectories = "true".equals(ElementParameterParser.getValue(node, "__INCLUDSUBDIR__"));
|
||||
|
||||
|
||||
final boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
log4jFileUtil.componentStartInfo(node);
|
||||
|
||||
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
IMetadataTable metadata = null;
|
||||
if ((metadatas!=null)&&(metadatas.size()>0)) {
|
||||
metadata = metadatas.get(0);
|
||||
metadata = metadatas.get(0);
|
||||
}
|
||||
|
||||
|
||||
List< ? extends IConnection> outputConnections = node.getOutgoingSortedConnections();
|
||||
|
||||
String dataOutputConnection = null;
|
||||
|
||||
String dataOutputConnection = null;
|
||||
for(IConnection conn : outputConnections) {
|
||||
if(conn.getLineStyle().hasConnectionCategory(IConnectionCategory.DATA)) {
|
||||
dataOutputConnection = conn.getName();
|
||||
break;
|
||||
} // if(conn) end
|
||||
} // for(conns) end
|
||||
|
||||
|
||||
%>
|
||||
<%
|
||||
if(!"true".equals(useExistingConn)){
|
||||
@@ -105,81 +105,115 @@ if(!"true".equals(useExistingConn)){
|
||||
<%=cid%>_client.authenticate(new com.box.boxjavalibv2.dao.BoxOAuthToken(<%=cid%>_map));
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
} else {
|
||||
%>
|
||||
com.box.boxjavalibv2.BoxClient <%=cid%>_client = (com.box.boxjavalibv2.BoxClient)globalMap.get(<%=connectionKey%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
String <%=cid%>_path = normalizePath(<%=path%>);
|
||||
com.box.boxjavalibv2.dao.BoxFolder <%=cid%>_rootFolder = <%=cid%>_client.getFoldersManager().getFolder("0", null);
|
||||
com.box.boxjavalibv2.dao.BoxItem <%=cid%>_boxItem;
|
||||
|
||||
int <%=cid%>_slashPosition = <%=cid%>_path.indexOf("/");
|
||||
String <%=cid%>_rootFolderName = <%=cid%>_slashPosition == -1 ? <%=cid%>_path : <%=cid%>_path.substring(0, <%=cid%>_slashPosition);
|
||||
if (<%=cid%>_rootFolderName.equals("All Files")) {
|
||||
List<String> <%=cid%>_pathItems = new java.util.ArrayList<String>(java.util.Arrays.asList(<%=cid%>_path.split("/")));
|
||||
<%=cid%>_pathItems.remove(0);
|
||||
try {
|
||||
<%=cid%>_boxItem = getBoxItemRecursively(<%=cid%>_pathItems, <%=cid%>_rootFolder, <%=cid%>_client);
|
||||
} catch (Exception e) {
|
||||
throw new Exception("<%=cid%> - " + "The path to file or directory was not found in " + <%=cid%>_path, e);
|
||||
}
|
||||
} else {
|
||||
throw new Exception("<%=cid%> - " + "The path to file or directory was not found in " + <%=cid%>_path);
|
||||
boolean <%=cid%>_hasError = false;
|
||||
String <%=cid%>_name = getFolderFromPath(<%=cid%>_path);
|
||||
com.box.boxjavalibv2.dao.BoxFile <%=cid%>_boxFile = null;
|
||||
com.box.boxjavalibv2.dao.BoxFolder <%=cid%>_boxFolder = null;
|
||||
if ((<%=cid%>_name.equals("All Files") && (<%=cid%>_path.isEmpty() || <%=cid%>_path == null)))
|
||||
{
|
||||
<%=cid%>_boxFolder = <%=cid%>_client.getFoldersManager().getFolder("0",null);
|
||||
}
|
||||
|
||||
java.util.List<com.box.boxjavalibv2.dao.BoxItem> <%=cid%>_items = new java.util.ArrayList<>();
|
||||
if (<%=cid%>_boxItem.getType().equals("folder")) {
|
||||
com.box.boxjavalibv2.dao.BoxFolder <%=cid%>_boxFolder = (com.box.boxjavalibv2.dao.BoxFolder) <%=cid%>_boxItem;
|
||||
<%if(includeSubDirectories) {%>
|
||||
<%if(filelistType.equalsIgnoreCase("Directories")) {%>
|
||||
<%=cid%>_items = getChildrenRecursively(<%=cid%>_client, <%=cid%>_boxFolder, <%=cid%>_items, true, false);
|
||||
<%} else if (filelistType.equalsIgnoreCase("Files")) {%>
|
||||
<%=cid%>_items = getChildrenRecursively(<%=cid%>_client, <%=cid%>_boxFolder, <%=cid%>_items, false, true);
|
||||
<%} else {%>
|
||||
<%=cid%>_items = getChildrenRecursively(<%=cid%>_client, <%=cid%>_boxFolder, <%=cid%>_items, true, true);
|
||||
<%}%>
|
||||
<%} else {%>
|
||||
for (com.box.boxjavalibv2.dao.BoxTypedObject obj : <%=cid%>_boxFolder.getItemCollection().getEntries())
|
||||
else
|
||||
{
|
||||
com.box.restclientv2.requestsbase.BoxDefaultRequestObject <%=cid%>_requestObject = new com.box.restclientv2.requestsbase.BoxDefaultRequestObject();
|
||||
if (<%=cid%>_name.equals("All Files") || <%=cid%>_path.equals("All Files"))
|
||||
{
|
||||
<%if(filelistType.equalsIgnoreCase("Directories")) {%>
|
||||
if(obj.getType().equals("folder"))
|
||||
{
|
||||
com.box.boxjavalibv2.dao.BoxFolder folder = <%=cid%>_client.getFoldersManager().getFolder(obj.getId(), null);
|
||||
<%=cid%>_items.add(folder);
|
||||
}
|
||||
<%} else if (filelistType.equalsIgnoreCase("Files")) {%>
|
||||
if (obj.getType().equals("file"))
|
||||
{
|
||||
com.box.boxjavalibv2.dao.BoxFile file = <%=cid%>_client.getFilesManager().getFile(obj.getId(), null);
|
||||
<%=cid%>_items.add(file);
|
||||
}
|
||||
<%} else {%>
|
||||
if(obj.getType().equals("folder"))
|
||||
{
|
||||
com.box.boxjavalibv2.dao.BoxFolder folder = <%=cid%>_client.getFoldersManager().getFolder(obj.getId(), null);
|
||||
<%=cid%>_items.add(folder);
|
||||
}
|
||||
else if (obj.getType().equals("file"))
|
||||
{
|
||||
com.box.boxjavalibv2.dao.BoxFile file = <%=cid%>_client.getFilesManager().getFile(obj.getId(), null);
|
||||
<%=cid%>_items.add(file);
|
||||
}
|
||||
<%}%>
|
||||
<%=cid%>_boxFolder = <%=cid%>_client.getFoldersManager().getFolder("0", null);
|
||||
}
|
||||
else
|
||||
{
|
||||
String parentPath = normalizePath(<%=cid%>_path.replace(<%=cid%>_name, ""));
|
||||
|
||||
<%=cid%>_boxFolder = <%=cid%>_client.getFoldersManager().getFolder("0", <%=cid%>_requestObject);
|
||||
|
||||
if ("All Files".equals(parentPath)) {
|
||||
|
||||
for (com.box.boxjavalibv2.dao.BoxTypedObject <%=cid%>_boxTypedObject : <%=cid%>_boxFolder.getItemCollection().getEntries()) {
|
||||
if (<%=cid%>_boxTypedObject.getValue("name").equals(<%=cid%>_name)) {
|
||||
if ("folder".equals(<%=cid%>_boxTypedObject.getType())) {
|
||||
<%=cid%>_boxFolder = <%=cid%>_client.getFoldersManager().getFolder(<%=cid%>_boxTypedObject.getId(), <%=cid%>_requestObject);
|
||||
} else {
|
||||
<%=cid%>_boxFile = <%=cid%>_client.getFilesManager().getFile(<%=cid%>_boxTypedObject.getId(),<%=cid%>_requestObject);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
List<String> <%=cid%>_paths = new java.util.ArrayList<String>(java.util.Arrays.asList(parentPath.split("/")));
|
||||
|
||||
<%=cid%>_boxFolder = getBoxFolderRecursively(<%=cid%>_paths, <%=cid%>_boxFolder, <%=cid%>_client);
|
||||
}
|
||||
}
|
||||
<%}%>
|
||||
} else if (<%=cid%>_boxItem.getType().equals("file")) {
|
||||
com.box.boxjavalibv2.dao.BoxFile <%=cid%>_boxFile = (com.box.boxjavalibv2.dao.BoxFile) <%=cid%>_boxItem;
|
||||
<%=cid%>_items.add(<%=cid%>_boxFile);
|
||||
}
|
||||
|
||||
for (com.box.boxjavalibv2.dao.BoxItem <%=cid%>_item : <%=cid%>_items) {
|
||||
String <%=cid%>_itemPath = getBoxItemPath(<%=cid%>_item);
|
||||
globalMap.put("<%=cid%>_NAME", <%=cid%>_item.getName());
|
||||
globalMap.put("<%=cid%>_ID", <%=cid%>_item.getId());
|
||||
globalMap.put("<%=cid%>_FILE_PATH", <%=cid%>_itemPath);
|
||||
globalMap.put("<%=cid%>_FILE_DIRECTORY", <%=cid%>_itemPath + <%=cid%>_item.getName());
|
||||
globalMap.put("<%=cid%>_TYPE",<%=cid%>_item.getType());
|
||||
globalMap.put("<%=cid%>_LAST_MODIFIED", <%=cid%>_item.getModifiedAt());
|
||||
globalMap.put("<%=cid%>_SIZE", <%=cid%>_item.getSize());
|
||||
if (<%=cid%>_boxFile == null && <%=cid%>_boxFolder == null)
|
||||
{
|
||||
throw new Exception("<%=cid%> - " + "No file or directory found in " + <%=path%>);
|
||||
}
|
||||
else if (<%=cid%>_boxFolder != null)
|
||||
{
|
||||
java.util.List<com.box.boxjavalibv2.dao.BoxItem> <%=cid%>_children = new java.util.ArrayList<com.box.boxjavalibv2.dao.BoxItem>();
|
||||
<%if(includeSubDirectories){%>
|
||||
<%if(filelistType.equalsIgnoreCase("Directories")){%>
|
||||
<%=cid%>_children = getChildrenRecursively(<%=cid%>_client, <%=cid%>_boxFolder, <%=cid%>_children, true, false);
|
||||
<%} else if (filelistType.equalsIgnoreCase("Files")){%>
|
||||
<%=cid%>_children = getChildrenRecursively(<%=cid%>_client, <%=cid%>_boxFolder, <%=cid%>_children, false, true);
|
||||
<%} else {%>
|
||||
<%=cid%>_children = getChildrenRecursively(<%=cid%>_client, <%=cid%>_boxFolder, <%=cid%>_children, true, true);
|
||||
<%}%>
|
||||
<%}%>
|
||||
<%else{%>
|
||||
for (com.box.boxjavalibv2.dao.BoxTypedObject obj : <%=cid%>_boxFolder.getItemCollection().getEntries())
|
||||
{
|
||||
<%if(filelistType.equalsIgnoreCase("Directories")){%>
|
||||
if(obj.getType().equals("folder"))
|
||||
{
|
||||
com.box.boxjavalibv2.dao.BoxFolder folder = <%=cid%>_client.getFoldersManager().getFolder(obj.getId(), null);
|
||||
<%=cid%>_children.add(folder);
|
||||
}
|
||||
<%} else if (filelistType.equalsIgnoreCase("Files")){%>
|
||||
if (obj.getType().equals("file"))
|
||||
{
|
||||
com.box.boxjavalibv2.dao.BoxFile file = <%=cid%>_client.getFilesManager().getFile(obj.getId(), null);
|
||||
<%=cid%>_children.add(file);
|
||||
}
|
||||
<%} else {%>
|
||||
if(obj.getType().equals("folder"))
|
||||
{
|
||||
com.box.boxjavalibv2.dao.BoxFolder folder = <%=cid%>_client.getFoldersManager().getFolder(obj.getId(), null);
|
||||
<%=cid%>_children.add(folder);
|
||||
}
|
||||
else if (obj.getType().equals("file"))
|
||||
{
|
||||
com.box.boxjavalibv2.dao.BoxFile file = <%=cid%>_client.getFilesManager().getFile(obj.getId(), null);
|
||||
<%=cid%>_children.add(file);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
|
||||
}
|
||||
<%}%>
|
||||
|
||||
for (com.box.boxjavalibv2.dao.BoxItem <%=cid%>_item : <%=cid%>_children)
|
||||
{
|
||||
String <%=cid%>_itemPath = getBoxItemPath(<%=cid%>_item);
|
||||
globalMap.put("<%=cid%>_NAME", <%=cid%>_item.getName());
|
||||
globalMap.put("<%=cid%>_ID", <%=cid%>_item.getId());
|
||||
globalMap.put("<%=cid%>_FILE_PATH", <%=cid%>_itemPath);
|
||||
globalMap.put("<%=cid%>_FILE_DIRECTORY", <%=cid%>_itemPath + <%=cid%>_item.getName());
|
||||
globalMap.put("<%=cid%>_TYPE",<%=cid%>_item.getType());
|
||||
globalMap.put("<%=cid%>_LAST_MODIFIED", <%=cid%>_item.getModifiedAt());
|
||||
globalMap.put("<%=cid%>_SIZE", <%=cid%>_item.getSize());
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
%>
|
||||
}
|
||||
|
||||
}
|
||||
//globalMap.put("<%=cid%>_NB_FILE", NB_FILE<%=cid%>);
|
||||
|
||||
|
||||
|
||||
@@ -898,8 +898,8 @@
|
||||
<IMPORT NAME="Driver-VERTICA_7.0.1" MODULE="vertica-jdbc-7.0.1-0.jar" MVN="mvn:org.talend.libraries/vertica-jdbc-7.0.1-0/6.0.0" REQUIRED_IF="(DBTYPE=='VERTICA') and (USE_EXISTING_CONNECTION == 'false') and (DB_VERTICA_VERSION=='VERTICA_7_0_X')" />
|
||||
<IMPORT NAME="Driver-VERTICA_7.1.2" MODULE="vertica-jdbc-7.1.2-0.jar" MVN="mvn:org.talend.libraries/vertica-jdbc-7.1.2-0/6.0.0" REQUIRED_IF="(DBTYPE=='VERTICA') and (USE_EXISTING_CONNECTION == 'false') and (DB_VERTICA_VERSION=='VERTICA_7_1_X')" />
|
||||
<IMPORT NAME="Driver-EXASolution" MODULE="exajdbc-6.0.9302.jar" MVN="mvn:org.talend.libraries/exajdbc-6.0.9302/6.3.0" REQUIRED_IF="(DBTYPE=='EXASOL') and (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-Redshift" MODULE="redshift-jdbc42-no-awssdk-1.2.32.1056.jar" MVN="mvn:com.amazon.redshift/redshift-jdbc42-no-awssdk/1.2.32.1056" REQUIRED_IF="(DBTYPE=='REDSHIFT') and (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-Snowflake" MODULE="snowflake-jdbc-3.11.1.jar" MVN="mvn:net.snowflake/snowflake-jdbc/3.11.1" REQUIRED_IF="(DBTYPE=='SNOWFLAKE') and (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-Redshift" MODULE="redshift-jdbc42-no-awssdk-1.2.20.1043.jar" MVN="mvn:com.amazon.redshift/redshift-jdbc42-no-awssdk/1.2.20.1043" REQUIRED_IF="(DBTYPE=='REDSHIFT') and (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-Snowflake" MODULE="snowflake-jdbc-3.5.3.jar" MVN="mvn:net.snowflake/snowflake-jdbc/3.5.3" REQUIRED_IF="(DBTYPE=='SNOWFLAKE') and (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-VERTICA_9.0" MODULE="vertica-jdbc-9.0.0-0.jar" MVN="mvn:org.talend.libraries/vertica-jdbc-9.0.0-0/6.0.0" REQUIRED_IF="(DBTYPE=='VERTICA') and (USE_EXISTING_CONNECTION == 'false') and (DB_VERTICA_VERSION=='VERTICA_9_0')"/>
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -2103,7 +2103,7 @@ class SnowflaketManager extends Manager {
|
||||
StringBuilder javaCode = new StringBuilder();
|
||||
if(useExistingConnection) {
|
||||
javaCode.append(retrieveTable());
|
||||
javaCode.append("java.sql.Connection conn_" + cid + " = (java.sql.Connection)globalMap.get(\"" + connection + "_connection\");");
|
||||
javaCode.append("java.sql.Connection conn_" + cid + " = (java.sql.Connection)globalMap.get(\"" + connection + "_Connection\");");
|
||||
} else {
|
||||
javaCode.append(super.getConnection());
|
||||
}
|
||||
|
||||
@@ -45,8 +45,8 @@ String xmlfile = ElementParameterParser.getValue(node, "__XMLFILE__");
|
||||
java.lang.StringBuilder sb<%=cid %>=new java.lang.StringBuilder("");
|
||||
|
||||
try{
|
||||
br<%=cid %> = new java.io.BufferedReader(new UnicodeReader(url<%=cid %>.openStream(), null));
|
||||
|
||||
br<%=cid %> = new java.io.BufferedReader(new java.io.InputStreamReader(url<%=cid %>.openStream()));
|
||||
|
||||
char[] buffer<%=cid %> = new char[1024];
|
||||
int length<%=cid %> = -1;
|
||||
while ((length<%=cid %> = br<%=cid %>.read(buffer<%=cid %>)) != -1)
|
||||
@@ -66,9 +66,7 @@ String xmlfile = ElementParameterParser.getValue(node, "__XMLFILE__");
|
||||
dbf<%=cid %> = javax.xml.parsers.DocumentBuilderFactory.newInstance();
|
||||
dbf<%=cid %>.setValidating(false);
|
||||
db<%=cid %> = dbf<%=cid %>.newDocumentBuilder();
|
||||
org.w3c.dom.Document doc<%=cid %> = db<%=cid %>.parse(
|
||||
new java.io.ByteArrayInputStream(sb<%=cid%>.toString().getBytes("UTF-8")));
|
||||
|
||||
org.w3c.dom.Document doc<%=cid %> = db<%=cid %>.parse(new java.io.StringBufferInputStream(sb<%=cid%>.toString()));
|
||||
String rootnode<%=cid %>=doc<%=cid %>.getDocumentElement().getNodeName();
|
||||
|
||||
String encoding=null;
|
||||
@@ -91,8 +89,8 @@ String xmlfile = ElementParameterParser.getValue(node, "__XMLFILE__");
|
||||
|
||||
int offsetRoot<%=cid %>=sb<%=cid %>.indexOf("<"+rootnode<%=cid %>);
|
||||
sb<%=cid%>.replace(0, offsetRoot<%=cid %>, reference<%=cid %>);
|
||||
is<%=cid %> = new java.io.ByteArrayInputStream(sb<%=cid%>.toString().getBytes("UTF-8"));
|
||||
|
||||
is<%=cid %>=new java.io.StringBufferInputStream(sb<%=cid %>.toString());
|
||||
|
||||
class MyHandler<%=cid %> extends org.xml.sax.helpers.DefaultHandler{
|
||||
String errorMessage = null;
|
||||
public void error(org.xml.sax.SAXParseException e) throws org.xml.sax.SAXException {
|
||||
|
||||
@@ -40,8 +40,8 @@
|
||||
</ADVANCED_PARAMETERS>
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT MODULE="janet-java-1.2.jar" MVN="mvn:org.talend.libraries/janet-java-1.2/6.0.0" NAME="janet-java-1.2"
|
||||
REQUIRED="true" />
|
||||
<IMPORT MODULE="janet-java-1.1.jar" MVN="mvn:org.talend.libraries/janet-java-1.1/6.0.0" NAME="janet-java-1.1"
|
||||
REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS>
|
||||
|
||||
@@ -94,7 +94,7 @@
|
||||
</ADVANCED_PARAMETERS>
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT MODULE="janet-java-1.2.jar" MVN="mvn:org.talend.libraries/janet-java-1.2/6.0.0" NAME="janet-java-1.2"
|
||||
<IMPORT MODULE="janet-java-1.1.jar" MVN="mvn:org.talend.libraries/janet-java-1.1/6.0.0" NAME="janet-java-1.1"
|
||||
REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
org.talend.designer.dbmap.language.mssql.MssqlGenerationManager gm = new org.talend.designer.dbmap.language.mssql.MssqlGenerationManager();
|
||||
org.talend.designer.dbmap.language.generation.GenericDbGenerationManager gm = new org.talend.designer.dbmap.language.generation.GenericDbGenerationManager();
|
||||
String uniqueNameComponent = null;
|
||||
IDbLanguage currentLanguage = gm.getLanguage();
|
||||
|
||||
|
||||
@@ -305,54 +305,48 @@ for(Column colStmt:stmtStructure){
|
||||
|
||||
if(("INSERT").equals(dataAction)){
|
||||
%>
|
||||
String insertQuery_<%=cid %> = "INSERT INTO "+tableName_<%=cid%>+"(<%=insertColName.toString()%>) ("+select_query_<%=cid %>+")";
|
||||
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(insertQuery_<%=cid %>);
|
||||
String insertQuery_<%=cid %> = "INSERT INTO "+tableName_<%=cid%>+"(<%=insertColName.toString()%>) ("+select_query_<%=cid %>+")";
|
||||
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(insertQuery_<%=cid %>);
|
||||
|
||||
<%
|
||||
}else if (("UPDATE").equals(dataAction)){
|
||||
if(useUpdateStatement){
|
||||
if(useUpdateStatement){
|
||||
%>
|
||||
String updateQuery_<%=cid %> = select_query_<%=cid %>;
|
||||
String updateQuery_<%=cid %> = select_query_<%=cid %>
|
||||
<%
|
||||
}else{
|
||||
}else{
|
||||
%>
|
||||
String updateQuery_<%=cid %> = "UPDATE "+tableName_<%=cid%>+" SET <%=updateSetStmt.toString()%> ";
|
||||
String updateQuery_<%=cid %> = "UPDATE "+tableName_<%=cid%>+" SET <%=updateSetStmt.toString()%> "
|
||||
<%
|
||||
}
|
||||
if(CodeGenerationUtils.hasAlphaNumericCharacter(whereClause)) {
|
||||
if(useUpdateStatement){
|
||||
}
|
||||
%>
|
||||
if(select_query_<%=cid %>.indexOf("WHERE")==-1){
|
||||
updateQuery_<%=cid %> +=" WHERE ";
|
||||
}else{
|
||||
updateQuery_<%=cid %> +=" AND ";
|
||||
}
|
||||
updateQuery_<%=cid %>+= <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>;
|
||||
<%
|
||||
}else{
|
||||
%>
|
||||
updateQuery_<%=cid %> += " WHERE " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>;
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(updateQuery_<%=cid %>);
|
||||
<%
|
||||
}else if (("DELETE").equals(dataAction)){
|
||||
%>
|
||||
String deleteQuery_<%=cid %> = "DELETE FROM "+ tableName_<%=cid%>+" WHERE EXISTS ("+select_query_<%=cid %>+") "
|
||||
<%
|
||||
if(CodeGenerationUtils.hasAlphaNumericCharacter(whereClause)) {
|
||||
%>
|
||||
+" AND " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>
|
||||
+" WHERE " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>
|
||||
<%
|
||||
}
|
||||
%>
|
||||
;
|
||||
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(deleteQuery_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
;
|
||||
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(updateQuery_<%=cid %>);
|
||||
<%
|
||||
}else if (("DELETE").equals(dataAction)){
|
||||
%>
|
||||
|
||||
String deleteQuery_<%=cid %> = "DELETE FROM "+ tableName_<%=cid%>+" WHERE EXISTS ("+select_query_<%=cid %>+") "
|
||||
<%
|
||||
if(CodeGenerationUtils.hasAlphaNumericCharacter(whereClause)) {
|
||||
%>
|
||||
+" AND " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>
|
||||
<%
|
||||
}
|
||||
%>
|
||||
;
|
||||
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(deleteQuery_<%=cid %>);
|
||||
|
||||
<%
|
||||
}
|
||||
%>
|
||||
int nb_line_<%=cid%> = 0;
|
||||
int nb_line_updated_<%=cid%> = 0;
|
||||
int nb_line_inserted_<%=cid%> = 0;
|
||||
|
||||
@@ -33,4 +33,4 @@ USE_DIFFERENT_TABLE.NAME=Use different table name
|
||||
DIFFERENT_TABLE_NAME.NAME=Table name
|
||||
IS_TABLE_NAME_VARIABLE.NAME=Table name from connection name is variable
|
||||
|
||||
USE_UPDATE_STATEMENT.NAME=Use update statement without subqueries
|
||||
USE_UPDATE_STATEMENT.NAME=Use update statement
|
||||
@@ -181,17 +181,6 @@
|
||||
</PARAMETER>
|
||||
|
||||
</PARAMETERS>
|
||||
|
||||
<ADVANCED_PARAMETERS>
|
||||
<PARAMETER
|
||||
NAME="PROPERTIES"
|
||||
FIELD="TEXT"
|
||||
NUM_ROW="1"
|
||||
REPOSITORY_VALUE="PROPERTIES_STRING"
|
||||
SHOW_IF="(#LINK@CONNECTOR.OUT.TABLE_REF == '0') AND (USE_EXISTING_CONNECTION == 'false')">
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
</ADVANCED_PARAMETERS>
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
|
||||
@@ -48,5 +48,4 @@ DB_VERSION.ITEM.MYSQL_5=Mysql 5
|
||||
DB_VERSION.ITEM.MYSQL_8=Mysql 8
|
||||
DB_VERSION.ITEM.MARIADB=MariaDB
|
||||
|
||||
PROPERTIES.NAME=Additional JDBC Parameters
|
||||
CONNECTION.NAME=Component List
|
||||
@@ -52,7 +52,6 @@ imports="
|
||||
String dbuser = null;
|
||||
String dbVersion = "";
|
||||
boolean useExistingConn = false;
|
||||
String dbproperties = null;
|
||||
if(previousNode != null) {
|
||||
dbhost = ElementParameterParser.getValue(previousNode, "__HOST__");
|
||||
dbport = ElementParameterParser.getValue(previousNode, "__PORT__");
|
||||
@@ -60,7 +59,6 @@ imports="
|
||||
dbuser = ElementParameterParser.getValue(previousNode, "__USER__");
|
||||
useExistingConn = ("true").equals(ElementParameterParser.getValue(previousNode, "__USE_EXISTING_CONNECTION__"));
|
||||
dbVersion = ElementParameterParser.getValue(previousNode, "__DB_VERSION__");
|
||||
dbproperties = ElementParameterParser.getValue(previousNode, "__PROPERTIES__");
|
||||
}
|
||||
|
||||
String differenttable = ElementParameterParser.getValue(node, "__DIFFERENT_TABLE_NAME__");
|
||||
@@ -98,11 +96,7 @@ imports="
|
||||
%>
|
||||
String driverClass_<%=cid%> = "<%=drverClass%>";
|
||||
java.lang.Class.forName(driverClass_<%=cid%>);
|
||||
<%if(dbproperties == null || ("\"\"").equals(dbproperties) || ("").equals(dbproperties)) {%>
|
||||
String url_<%=cid %> = "<%=jdbcURL%>://"+<%=dbhost%>+":"+<%=dbport%>+"/"+<%=dbname%>;
|
||||
<%} else {%>
|
||||
String url_<%=cid %> = "<%=jdbcURL%>://"+<%=dbhost%>+":"+<%=dbport%>+"/"+<%=dbname%>+"?"+<%=dbproperties%>;
|
||||
<%}%>
|
||||
String dbUser_<%=cid%> = <%=dbuser%>;
|
||||
|
||||
<%
|
||||
|
||||
@@ -140,15 +140,7 @@
|
||||
|
||||
</PARAMETERS>
|
||||
|
||||
<ADVANCED_PARAMETERS>
|
||||
<PARAMETER
|
||||
NAME="USE_UPDATE_STATEMENT"
|
||||
FIELD="CHECK"
|
||||
NUM_ROW="10"
|
||||
SHOW_IF="DATA_ACTION=='UPDATE'"
|
||||
>
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
<ADVANCED_PARAMETERS>
|
||||
</ADVANCED_PARAMETERS>
|
||||
|
||||
<CODEGENERATION>
|
||||
|
||||
@@ -35,7 +35,6 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
String differenttable = ElementParameterParser.getValue(node, "__DIFFERENT_TABLE_NAME__");
|
||||
boolean useDifferentTable = "true".equals(ElementParameterParser.getValue(node, "__USE_DIFFERENT_TABLE__"));
|
||||
boolean isTableNameVariable="true".equals(ElementParameterParser.getValue(node, "__IS_TABLE_NAME_VARIABLE__"));
|
||||
boolean useUpdateStatement="true".equals(ElementParameterParser.getValue(node, "__USE_UPDATE_STATEMENT__"));
|
||||
%>
|
||||
String select_query_<%=cid %> = null;
|
||||
String tableName_<%=cid%> = null;
|
||||
@@ -94,7 +93,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
String username = "username_" + connection;
|
||||
String connectionKey=null;
|
||||
if("tSnowflakeConnection".equals(connNode.getComponent().getName())){
|
||||
connectionKey=connection+"_connection";
|
||||
connectionKey=connection+"_Connection";
|
||||
}else{
|
||||
connectionKey=conn;
|
||||
}
|
||||
@@ -157,36 +156,21 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
|
||||
if(("INSERT").equals(dataAction)){
|
||||
%>
|
||||
String insertQuery_<%=cid %> = "INSERT INTO "+tableName_<%=cid%>+"(<%=insertColName.toString()%>) ("+select_query_<%=cid %>+")";
|
||||
<%
|
||||
} else if (("UPDATE").equals(dataAction)){
|
||||
if(useUpdateStatement){
|
||||
%>
|
||||
String updateQuery_<%=cid %> = select_query_<%=cid %>;
|
||||
<%
|
||||
if(CodeGenerationUtils.hasAlphaNumericCharacter(whereClause)) {
|
||||
%>
|
||||
if (updateQuery_<%=cid %>.toUpperCase().contains(" WHERE ")) {
|
||||
updateQuery_<%=cid %> += " AND (" + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%> + ")";
|
||||
} else {
|
||||
updateQuery_<%=cid %> += " WHERE " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>;
|
||||
}
|
||||
<%
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
String updateQuery_<%=cid %> = "UPDATE "+tableName_<%=cid%>+" SET <%=updateSetStmt.toString()%> "
|
||||
<%
|
||||
if(CodeGenerationUtils.hasAlphaNumericCharacter(whereClause)) {
|
||||
%>
|
||||
+" WHERE " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>
|
||||
<%
|
||||
}
|
||||
%>
|
||||
;
|
||||
%>
|
||||
String insertQuery_<%=cid %> = "INSERT INTO "+tableName_<%=cid%>+"(<%=insertColName.toString()%>) ("+select_query_<%=cid %>+")";
|
||||
<%
|
||||
} else if (("UPDATE").equals(dataAction)){
|
||||
%>
|
||||
String updateQuery_<%=cid %> = "UPDATE "+tableName_<%=cid%>+" SET <%=updateSetStmt.toString()%> "
|
||||
<%
|
||||
if(CodeGenerationUtils.hasAlphaNumericCharacter(whereClause)) {
|
||||
%>
|
||||
+" WHERE " + <%=CodeGenerationUtils.replaceAllCrBySpace(whereClause)%>
|
||||
<%
|
||||
}
|
||||
%>
|
||||
;
|
||||
<%
|
||||
}
|
||||
} else if (("DELETE").equals(dataAction)){
|
||||
%>
|
||||
String deleteQuery_<%=cid %> = "DELETE FROM "+ tableName_<%=cid%>+" WHERE EXISTS ("+select_query_<%=cid %>+") "
|
||||
|
||||
@@ -40,6 +40,4 @@ TABLE_ACTION.ITEM.CREATE_IF_NOT_EXISTS=Create table if not exists
|
||||
TABLE_ACTION.ITEM.DROP_IF_EXISTS_AND_CREATE=Drop table if exists and create
|
||||
TABLE_ACTION.ITEM.DROP_CREATE=Drop and create table
|
||||
TABLE_ACTION.ITEM.NONE=None
|
||||
TABLE_ACTION.NAME=Action on table
|
||||
|
||||
USE_UPDATE_STATEMENT.NAME=Use update statement without subqueries
|
||||
TABLE_ACTION.NAME=Action on table
|
||||
@@ -107,7 +107,7 @@ imports="
|
||||
}
|
||||
else{
|
||||
%>
|
||||
String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName=" + <%=dbname %> + ";" + <%=dbproperties%>;
|
||||
String url_<%=cid%> = "jdbc:sqlanywhere:Host=" + <%=dbhost %> + ":" + <%=dbport %> + ";DatabaseName" + <%=dbname %> + ";" + <%=dbproperties%>;
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,7 +178,7 @@
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Java_DOM4J1.6" MODULE="dom4j-1.6.1.jar" MVN="mvn:dom4j/dom4j/1.6.1" UrlPath="platform:/plugin/org.talend.libraries.dom4j-jaxen/lib/dom4j-1.6.1.jar" REQUIRED_IF="READ_BY == 'XPATH'" BundleID="" />
|
||||
<IMPORT NAME="Java_JAXEN1.1" MODULE="jaxen-1.1.1.jar" MVN="mvn:org.talend.libraries/jaxen-1.1.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.dom4j-jaxen/lib/jaxen-1.1.1.jar" REQUIRED_IF="READ_BY == 'XPATH'" BundleID="" />
|
||||
<IMPORT NAME="json-lib" MODULE="json-lib-2.4.5-talend.jar" MVN="mvn:net.sf.json-lib/json-lib/2.4.5-talend" REQUIRED_IF="READ_BY == 'XPATH'" />
|
||||
<IMPORT NAME="json-lib" MODULE="json-lib-2.4.2-talend.jar" MVN="mvn:net.sf.json-lib/json-lib/2.4.2-talend" REQUIRED_IF="READ_BY == 'XPATH'" />
|
||||
<IMPORT NAME="commons_lang" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED_IF="READ_BY == 'XPATH'" />
|
||||
<IMPORT NAME="commons_logging" MODULE="commons-logging-1.1.1.jar" MVN="mvn:org.talend.libraries/commons-logging-1.1.1/6.0.0" UrlPath="platform:/base/plugins/org.apache.commons.logging_1.1.1.v201101211721.jar" REQUIRED_IF="READ_BY == 'XPATH'" />
|
||||
<IMPORT NAME="ezmorph" MODULE="ezmorph-1.0.6.jar" MVN="mvn:org.talend.libraries/ezmorph-1.0.6/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.jackson/lib/ezmorph-1.0.6.jar" REQUIRED_IF="READ_BY == 'XPATH'" />
|
||||
|
||||
@@ -149,7 +149,7 @@
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Talen File Enhanced"
|
||||
MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar" REQUIRED="true" />
|
||||
MODULE="talend_file_enhanced_20070724.jar" MVN="mvn:org.talend.libraries/talend_file_enhanced_20070724/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced_20070724.jar" REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -17,16 +17,20 @@
|
||||
if (connObj != null) {
|
||||
try {
|
||||
|
||||
<% if (sftp) { %>
|
||||
<% if (sftp == false && ftps == false) { %>
|
||||
com.enterprisedt.net.ftp.FTPClient conn = (com.enterprisedt.net.ftp.FTPClient) connObj;
|
||||
conn.executeCommand("quit");
|
||||
<% } else if (sftp) { %>
|
||||
com.jcraft.jsch.ChannelSftp channel = (com.jcraft.jsch.ChannelSftp) connObj;
|
||||
com.jcraft.jsch.Session session = channel.getSession();
|
||||
channel.disconnect();
|
||||
session.disconnect();
|
||||
<% } else { %>
|
||||
org.apache.commons.net.ftp.FTPClient conn = (org.apache.commons.net.ftp.FTPClient) connObj;
|
||||
<% } else if (ftps) { %>
|
||||
org.apache.commons.net.ftp.FTPSClient conn = (org.apache.commons.net.ftp.FTPSClient) connObj;
|
||||
conn.disconnect();
|
||||
<% }
|
||||
%>
|
||||
<% } else { %>
|
||||
System.err.println("Invalid tFTPConnection configuration.");
|
||||
<% } %>
|
||||
|
||||
} catch (Exception e) {
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE", e.getMessage());
|
||||
|
||||
@@ -15,13 +15,16 @@
|
||||
%>
|
||||
Object connObj = globalMap.get("conn_<%=connection%>");
|
||||
if (connObj != null) {
|
||||
<% if (sftp) { %>
|
||||
<% if (sftp == false && ftps == false) { %>
|
||||
com.enterprisedt.net.ftp.FTPClient conn = (com.enterprisedt.net.ftp.FTPClient) connObj;
|
||||
conn.executeCommand("quit");
|
||||
<% } else if (sftp) { %>
|
||||
com.jcraft.jsch.ChannelSftp channel = (com.jcraft.jsch.ChannelSftp) connObj;
|
||||
com.jcraft.jsch.Session session = channel.getSession();
|
||||
channel.disconnect();
|
||||
session.disconnect();
|
||||
<% } else { %>
|
||||
org.apache.commons.net.ftp.FTPClient conn = (org.apache.commons.net.ftp.FTPClient) connObj;
|
||||
<% } else if (ftps) { %>
|
||||
org.apache.commons.net.ftp.FTPSClient conn = (org.apache.commons.net.ftp.FTPSClient) connObj;
|
||||
conn.disconnect();
|
||||
<% } %>
|
||||
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
<%@ jet
|
||||
%>
|
||||
|
||||
boolean retry_<%=cid%> = false;
|
||||
int retry_count_<%=cid%> = 0;
|
||||
int retry_max_<%=cid%> = 5;
|
||||
|
||||
com.jcraft.jsch.Session session_<%=cid%> = null;
|
||||
com.jcraft.jsch.Channel channel_<%=cid%> = null;
|
||||
|
||||
do {
|
||||
retry_<%=cid%> = false;
|
||||
|
||||
com.jcraft.jsch.JSch jsch_<%=cid%> = new com.jcraft.jsch.JSch();
|
||||
|
||||
<%if (("PUBLICKEY").equals(authMethod)){%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - SFTP authentication using a public key.");
|
||||
log.debug("<%=cid%> - Private key: '" + <%=privateKey%> + "'.");
|
||||
<%}%>
|
||||
jsch_<%=cid%>.addIdentity(<%=privateKey %>, defaultUserInfo_<%=cid%>.getPassphrase());
|
||||
<%}%>
|
||||
|
||||
session_<%=cid%> = jsch_<%=cid%>.getSession(<%=user%>, <%=host%>, <%=port%>);
|
||||
session_<%=cid%>.setConfig("PreferredAuthentications", "publickey,password,keyboard-interactive,gssapi-with-mic");
|
||||
|
||||
<%if (("PASSWORD").equals(authMethod)) {%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - SFTP authentication using a password.");
|
||||
<%}%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
session_<%=cid%>.setPassword(decryptedPassword_<%=cid%>);
|
||||
<%}%>
|
||||
|
||||
session_<%=cid%>.setUserInfo(defaultUserInfo_<%=cid%>);
|
||||
<%if (!useProxy) {%>
|
||||
if(("true").equals(System.getProperty("http.proxySet")) ){
|
||||
com.jcraft.jsch.ProxyHTTP proxy_<%=cid%> = new com.jcraft.jsch.ProxyHTTP(System.getProperty("http.proxyHost"),Integer.parseInt(System.getProperty("http.proxyPort")));
|
||||
if(!"".equals(System.getProperty("http.proxyUser"))){
|
||||
proxy_<%=cid%>.setUserPasswd(System.getProperty("http.proxyUser"),System.getProperty("http.proxyPassword"));
|
||||
}
|
||||
session_<%=cid%>.setProxy(proxy_<%=cid%>);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user%> + "'.");
|
||||
<%}%>
|
||||
|
||||
channel_<%=cid%> = null;
|
||||
try {
|
||||
session_<%=cid%>.connect();
|
||||
channel_<%=cid%> = session_<%=cid%>.openChannel("sftp");
|
||||
channel_<%=cid%>.connect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
} catch (com.jcraft.jsch.JSchException e_<%=cid%>) {
|
||||
try {
|
||||
if(channel_<%=cid%>!=null) {
|
||||
channel_<%=cid%>.disconnect();
|
||||
}
|
||||
|
||||
if(session_<%=cid%>!=null) {
|
||||
session_<%=cid%>.disconnect();
|
||||
}
|
||||
} catch(java.lang.Exception ce_<%=cid%>) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - close sftp connection failed : " + ce_<%=cid%>.getClass() + " : " + ce_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
}
|
||||
|
||||
String message_<%=cid%> = new TalendException(null, null, null).getExceptionCauseMessage(e_<%=cid%>);
|
||||
if(message_<%=cid%>.contains("Signature length not correct") || message_<%=cid%>.contains("connection is closed by foreign host")) {
|
||||
retry_<%=cid%> = true;
|
||||
retry_count_<%=cid%>++;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - connect: Signature length not correct or connection is closed by foreign host, so retry, retry time : " + retry_count_<%=cid%>);
|
||||
<%}%>
|
||||
} else {
|
||||
throw e_<%=cid%>;
|
||||
}
|
||||
}
|
||||
} while(retry_<%=cid%> && (retry_count_<%=cid%> < retry_max_<%=cid%>));
|
||||
|
||||
com.jcraft.jsch.ChannelSftp c_<%=cid%> = (com.jcraft.jsch.ChannelSftp)channel_<%=cid%>;
|
||||
@@ -28,7 +28,7 @@ String authMethod = ElementParameterParser.getValue(node, "__AUTH_METHOD__");
|
||||
String privateKey = ElementParameterParser.getValue(node, "__PRIVATEKEY__");
|
||||
boolean ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String encoding = ElementParameterParser.getValue(node, "__ENCODING__");
|
||||
String sEncoding = ElementParameterParser.getValue(node, "__ENCODING__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
String protectionLevel = ElementParameterParser.getValue(node, "__FTPS_PROT__");
|
||||
String protectionBufferSize = ElementParameterParser.getValue(node, "__FTPS_PROTECTION_BUFF_SIZE__");
|
||||
@@ -60,22 +60,42 @@ if (ElementParameterParser.canEncrypt(node, passwordFieldName)) {
|
||||
});
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%
|
||||
if (!sftp && !ftps) { // *** ftp *** //
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
|
||||
<%
|
||||
passwordFieldName = "__PASS__";
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
|
||||
|
||||
globalMap.put("conn_<%=cid%>",ftp_<%=cid %>);
|
||||
if (!sftp && !ftps) { // *** ftp *** //
|
||||
if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
|
||||
<%}%>
|
||||
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
|
||||
ftp_<%=cid %>.setRemoteHost(<%=host %>);
|
||||
ftp_<%=cid %>.setRemotePort(<%=port %>);
|
||||
|
||||
<%if (("ACTIVE").equals(connectMode)) {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
|
||||
<%} else {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setControlEncoding(<%=sEncoding%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect();
|
||||
|
||||
<%
|
||||
passwordFieldName = "__PASS__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
globalMap.put("conn_<%=cid%>",ftp_<%=cid %>);
|
||||
<%
|
||||
} else if (!ftps) { // *** sftp *** //
|
||||
%>
|
||||
class MyUserInfo implements com.jcraft.jsch.UserInfo, com.jcraft.jsch.UIKeyboardInteractive {
|
||||
|
||||
|
||||
<%
|
||||
passwordFieldName = "__PASSPHRASE__";
|
||||
if (ElementParameterParser.canEncrypt(node, passwordFieldName)) {
|
||||
@@ -118,20 +138,99 @@ if (!sftp && !ftps) { // *** ftp *** //
|
||||
passwordFieldName = "__PASS__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
|
||||
<%@ include file="sftp_connect.javajet"%>
|
||||
|
||||
<%if(use_encoding) {%>
|
||||
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
|
||||
c_<%=cid%>.setFilenameEncoding(<%=sEncoding%>);
|
||||
<%}%>
|
||||
|
||||
globalMap.put("conn_<%=cid%>", c_<%=cid%>);
|
||||
<%
|
||||
} else { // *** ftps *** //
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
|
||||
globalMap.put("conn_<%=cid%>", ftp_<%=cid %>);
|
||||
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
|
||||
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
|
||||
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
|
||||
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
MyTrust_<%=cid%> myTrust_<%=cid%> = null;
|
||||
|
||||
try {
|
||||
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
|
||||
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
|
||||
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
|
||||
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
|
||||
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
|
||||
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
|
||||
if (isHTTPProxyUsed) {
|
||||
String httpProxyHost = System.getProperty("http.proxyHost");
|
||||
int httpProxyPort = Integer.getInteger("http.proxyPort");
|
||||
String httpProxyUser = System.getProperty("http.proxyUser");
|
||||
String httpProxyPass = System.getProperty("http.proxyPassword");
|
||||
|
||||
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
|
||||
} else {
|
||||
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
|
||||
}
|
||||
ftp_<%=cid %>.setControlEncoding(<%=sEncoding %>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
|
||||
<%
|
||||
passwordFieldName = "__PASS__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
|
||||
if (!isLoginSuccessful_<%=cid %>) {
|
||||
throw new RuntimeException("Login failed");
|
||||
}
|
||||
|
||||
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
|
||||
<%
|
||||
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
|
||||
<%
|
||||
}
|
||||
|
||||
if (protectionLevel != null && !protectionLevel.isEmpty()) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} catch (Exception e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
|
||||
<%}%>
|
||||
throw e;
|
||||
}
|
||||
|
||||
<%
|
||||
if ("PASSIVE".equals(connectMode)) {
|
||||
%>
|
||||
ftp_<%=cid %>.enterLocalPassiveMode();
|
||||
<%
|
||||
if (isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - Using the passive mode.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
globalMap.put("conn_<%=cid%>", ftp_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -185,6 +185,7 @@
|
||||
REQUIRED="true" NOT_SHOW_IF="(SFTP == 'true') and (USE_ENCODING == 'false')">
|
||||
<DEFAULT>"ISO-8859-15"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
</PARAMETERS>
|
||||
|
||||
<ADVANCED_PARAMETERS>
|
||||
@@ -235,10 +236,10 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false')"/>
|
||||
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true')"/>
|
||||
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false')" />
|
||||
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar" MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819" REQUIRED_IF="(FTPS == 'true')" />
|
||||
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED="true" />
|
||||
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true')" />
|
||||
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar" MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819" REQUIRED_IF="(FTPS == 'true')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -55,8 +55,6 @@ if (("true").equals(useExistingConn)) {
|
||||
} else {
|
||||
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
|
||||
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
|
||||
%>
|
||||
<%
|
||||
}
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
@@ -131,7 +129,7 @@ int nb_file_<%=cid%> = 0;
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
|
||||
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
|
||||
|
||||
<%if(use_encoding) {%>
|
||||
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
|
||||
@@ -168,70 +166,191 @@ int nb_file_<%=cid%> = 0;
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
|
||||
|
||||
<%
|
||||
} else {
|
||||
if (ftps) { // *** ftps *** //
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
<%
|
||||
} else if (ftps) { // *** ftps *** //
|
||||
if (("false").equals(useExistingConn)) {
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
|
||||
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
|
||||
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
|
||||
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
|
||||
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
|
||||
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
|
||||
|
||||
try {
|
||||
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
|
||||
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
|
||||
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
|
||||
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
|
||||
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
|
||||
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
|
||||
if (isHTTPProxyUsed) {
|
||||
String httpProxyHost = System.getProperty("http.proxyHost");
|
||||
int httpProxyPort = Integer.getInteger("http.proxyPort");
|
||||
String httpProxyUser = System.getProperty("http.proxyUser");
|
||||
String httpProxyPass = System.getProperty("http.proxyPassword");
|
||||
|
||||
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
|
||||
} else {
|
||||
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
|
||||
}
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
|
||||
if (!isLoginSuccessful_<%=cid %>) {
|
||||
throw new RuntimeException("Login failed");
|
||||
}
|
||||
|
||||
<%
|
||||
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
|
||||
<%
|
||||
}
|
||||
|
||||
if (protectionLevel != null && !protectionLevel.isEmpty()) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} catch (Exception e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
|
||||
<%}%>
|
||||
<%
|
||||
if (dieOnError) {
|
||||
%>
|
||||
throw e;
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
e.printStackTrace();
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
|
||||
<%
|
||||
if ("PASSIVE".equals(connectMode)) {
|
||||
%>
|
||||
ftp_<%=cid %>.enterLocalPassiveMode();
|
||||
<%
|
||||
if (isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - Using the passive mode.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
<%
|
||||
if (moveToCurrentDir) {
|
||||
%>
|
||||
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(<%=remotedir %>);
|
||||
|
||||
if (!cwdSuccess_<%=cid %>) {
|
||||
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
<% /*store real pwd*/ %>
|
||||
String rootWorkDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<>();
|
||||
<%
|
||||
} else {// *** ftp *** //
|
||||
for (int i=0; i<files.size(); i++) {
|
||||
Map<String, String> line = files.get(i);
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> =null;
|
||||
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
|
||||
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,"");
|
||||
list<%=cid %>.add(map<%=cid %><%=i %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
|
||||
<%
|
||||
} else {// *** ftp *** //
|
||||
%>
|
||||
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> =null;
|
||||
|
||||
<%if (("true").equals(useExistingConn)) {%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
|
||||
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Uses an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
log.info("<%=cid%> - Uses an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ",Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%if(!moveToCurrentDir){%>
|
||||
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
|
||||
<%}%>
|
||||
<%} else {%>
|
||||
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
|
||||
ftp_<%=cid %>.setRemoteHost(<%=host %>);
|
||||
ftp_<%=cid %>.setRemotePort(<%=port %>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
|
||||
<%}%>
|
||||
<%if (("ACTIVE").equals(connectMode)) {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
|
||||
<%} else {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username: '" +<%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect();
|
||||
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
<%}%>
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
|
||||
|
||||
<%} else {
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
|
||||
<%}%>
|
||||
<%
|
||||
}//common code ftp and ftps:
|
||||
%>
|
||||
<%
|
||||
if (("true").equals(useExistingConn) && moveToCurrentDir) {
|
||||
%>
|
||||
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(<%=remotedir %>);
|
||||
<%for (int i = 0; i < files.size(); i++) {
|
||||
Map<String, String> line = files.get(i);
|
||||
%>
|
||||
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
|
||||
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,"");
|
||||
list<%=cid %>.add(map<%=cid %><%=i %>);
|
||||
<%}%>
|
||||
String remotedir<%=cid %> = <%=remotedir%>;
|
||||
ftp_<%=cid %>.chdir(remotedir<%=cid %>);
|
||||
String root<%=cid %> = ftp_<%=cid %>.pwd();
|
||||
|
||||
if (!cwdSuccess_<%=cid %>) {
|
||||
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<% /*store real pwd*/ %>
|
||||
String rootWorkDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<>();
|
||||
<%
|
||||
for (int i=0; i<files.size(); i++) {
|
||||
Map<String, String> line = files.get(i);
|
||||
%>
|
||||
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
|
||||
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,"");
|
||||
list<%=cid %>.add(map<%=cid %><%=i %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
<%if("FILE".equals(targetType)){%>
|
||||
log.info("<%=cid%> - Deleting file from server.");
|
||||
<%}else{%>
|
||||
log.info("<%=cid%> - Deleting directory from server.");
|
||||
<%}%>
|
||||
<%}%>
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
|
||||
<%}%>
|
||||
|
||||
@@ -70,14 +70,14 @@
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
}catch(java.io.IOException ioe_<%=cid%>){
|
||||
}catch(java.net.SocketException se_<%=cid%>){
|
||||
//ignore failure
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
}
|
||||
<%
|
||||
@@ -86,12 +86,18 @@
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
}else{
|
||||
if(!moveToCurrentDir){
|
||||
%>
|
||||
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -177,7 +177,7 @@
|
||||
<DEFAULT>"."</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="31" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="31" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
|
||||
<DEFAULT>true</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
@@ -292,13 +292,13 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="jakarta-oro" MODULE="jakarta-oro-2.0.8.jar" MVN="mvn:org.talend.libraries/jakarta-oro-2.0.8/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache/lib/jakarta-oro-2.0.8.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
|
||||
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS>
|
||||
|
||||
@@ -95,7 +95,7 @@
|
||||
|
||||
|
||||
<%
|
||||
}else { // *** ftp, ftps *** //
|
||||
}else if (ftps) { // *** ftps *** //
|
||||
%>
|
||||
for (String key<%=cid %> : keySet<%=cid %>) {
|
||||
<% /*prepare filemask*/ %>
|
||||
@@ -241,6 +241,73 @@
|
||||
|
||||
<% /*return to root dir for next filemask*/ %>
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootWorkDir_<%=cid %>);
|
||||
<%
|
||||
}else {// *** ftp *** //
|
||||
%>
|
||||
try {
|
||||
for (String key<%=cid %> : keySet<%=cid %>) {
|
||||
String filemask<%=cid %> = key<%=cid %>;
|
||||
String dir<%=cid %> = null;
|
||||
<%if(!usePerl5) {//perl5 mode not support windows(\) path separator at the mask string%>
|
||||
String mask<%=cid %> = filemask<%=cid %>.replaceAll("\\\\", "/") ;
|
||||
<%}else{%>
|
||||
String mask<%=cid %> = filemask<%=cid %>;
|
||||
<%}%>
|
||||
int i<%=cid %> = mask<%=cid %>.lastIndexOf('/');
|
||||
if (i<%=cid %>!=-1) {
|
||||
dir<%=cid %> = mask<%=cid %>.substring(0, i<%=cid %>);
|
||||
mask<%=cid %> = mask<%=cid %>.substring(i<%=cid %>+1);
|
||||
}
|
||||
if (dir<%=cid %>!=null && !"".equals(dir<%=cid %>)) ftp_<%=cid %>.chdir(dir<%=cid %>);
|
||||
<%if(!usePerl5) {%>
|
||||
mask<%=cid %> = org.apache.oro.text.GlobCompiler.globToPerl5(mask<%=cid %>.toCharArray(), org.apache.oro.text.GlobCompiler.DEFAULT_MASK);
|
||||
<%}%>
|
||||
String[] fileName_<%=cid%> = null;
|
||||
boolean dirFailed_<%=cid%> = false;
|
||||
try{
|
||||
fileName_<%=cid%> = ftp_<%=cid%>.dir(null,false);
|
||||
}catch(com.enterprisedt.net.ftp.FTPException e_<%=cid%>){
|
||||
dirFailed_<%=cid%> = true;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + e_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
}
|
||||
if((fileName_<%=cid%> == null) || dirFailed_<%=cid%> || (fileName_<%=cid%>.length <= 0)){
|
||||
com.enterprisedt.net.ftp.FTPFile[] fileList_<%=cid%> = ftp_<%=cid%>.dirDetails(null);
|
||||
fileName_<%=cid%> = new String[fileList_<%=cid%>.length];
|
||||
for (int i_<%=cid%> = 0 ; i_<%=cid%> < fileList_<%=cid%>.length ; i_<%=cid%>++){
|
||||
fileName_<%=cid%>[i_<%=cid%>] = fileList_<%=cid%>[i_<%=cid%>].getName();
|
||||
}
|
||||
}
|
||||
for (String file_<%=cid%> : fileName_<%=cid%>) {
|
||||
if (file_<%=cid%>.matches(mask<%=cid %>)) {
|
||||
<%
|
||||
if("FILE".equals(targetType)){
|
||||
%>
|
||||
ftp_<%=cid %>.delete(file_<%=cid%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - File '" + file_<%=cid%> + "' was deleted from the remote directory " + <%=remotedir %> + ".");
|
||||
<%}%>
|
||||
<%
|
||||
}else{
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Directory '" + file_<%=cid%> + "' was deleted from the remote directory " + <%=remotedir %> + ".");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.rmdir(file_<%=cid%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File deleted.");
|
||||
nb_file_<%=cid%>++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(com.enterprisedt.net.ftp.FTPException e_<%=cid %>){
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "Deleting file action error");
|
||||
throw e_<%=cid %>;
|
||||
}
|
||||
ftp_<%=cid %>.chdir(root<%=cid %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -57,8 +57,6 @@ if (("true").equals(useExistingConn)) {
|
||||
} else {
|
||||
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
|
||||
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
|
||||
%>
|
||||
<%
|
||||
}
|
||||
|
||||
String passwordFieldName = "";
|
||||
@@ -136,7 +134,7 @@ if (sftp) { // *** sftp *** //
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
|
||||
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
|
||||
|
||||
<%if(use_encoding) {%>
|
||||
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
|
||||
@@ -155,38 +153,134 @@ if (sftp) { // *** sftp *** //
|
||||
}
|
||||
<%
|
||||
}
|
||||
} else { // *** ftp+s *** //
|
||||
if (ftps) {
|
||||
} else if (ftps) { // *** ftps *** //
|
||||
if (("false").equals(useExistingConn)) {
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
|
||||
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
|
||||
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
|
||||
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
|
||||
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
|
||||
|
||||
try {
|
||||
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
|
||||
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
|
||||
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
|
||||
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
|
||||
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
|
||||
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
|
||||
if (isHTTPProxyUsed) {
|
||||
String httpProxyHost = System.getProperty("http.proxyHost");
|
||||
int httpProxyPort = Integer.getInteger("http.proxyPort");
|
||||
String httpProxyUser = System.getProperty("http.proxyUser");
|
||||
String httpProxyPass = System.getProperty("http.proxyPassword");
|
||||
|
||||
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
|
||||
} else {
|
||||
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
|
||||
}
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
|
||||
<%
|
||||
if (("false").equals(useExistingConn)) {
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
|
||||
if (!isLoginSuccessful_<%=cid %>) {
|
||||
throw new RuntimeException("Login failed");
|
||||
}
|
||||
|
||||
<%
|
||||
} else {
|
||||
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
|
||||
<%
|
||||
}
|
||||
} else { // *** ftp *** //
|
||||
}
|
||||
|
||||
if (protectionLevel != null && !protectionLevel.isEmpty()) {
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
|
||||
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
|
||||
<%
|
||||
if (("true").equals(useExistingConn)) {%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%} else {
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
}
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
|
||||
} catch (Exception e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
|
||||
<%}%>
|
||||
throw e;
|
||||
}
|
||||
<%
|
||||
}
|
||||
if ("PASSIVE".equals(connectMode)) {
|
||||
%>
|
||||
ftp_<%=cid %>.enterLocalPassiveMode();
|
||||
<%
|
||||
if (isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - Using the passive mode.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
<%
|
||||
}
|
||||
} else { // *** ftp *** //
|
||||
%>
|
||||
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> = null;
|
||||
|
||||
<% if (("true").equals(useExistingConn)) {%>
|
||||
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%} else {%>
|
||||
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
|
||||
ftp_<%=cid %>.setRemoteHost(<%=host %>);
|
||||
ftp_<%=cid %>.setRemotePort(<%=port %>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
|
||||
<%}%>
|
||||
|
||||
<%if (("ACTIVE").equals(connectMode)) {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
|
||||
<%} else {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username: '" +<%=username %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect();
|
||||
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
ftp_<%=cid %>.login(<%=username %>, decryptedPassword_<%=cid%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host%> + "' has succeeded.");
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
}%>
|
||||
|
||||
@@ -56,35 +56,41 @@ if (sftp) { // *** sftp *** //
|
||||
<%
|
||||
}
|
||||
} else { // *** ftp *** //
|
||||
if (!("true").equals(useExistingConn)) {
|
||||
if (("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
try {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
} catch(java.io.IOException ioe_<%=cid%>) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
//ignore failure
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
if (!("true").equals(useExistingConn)) {
|
||||
if (("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
try {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
} catch(java.net.SocketException se_<%=cid%>) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
//ignore failure
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
} else {
|
||||
if (moveToCurrentDir) {
|
||||
%>
|
||||
ftp_<%=cid %>.chdir(<%=remoteDir %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -170,7 +170,7 @@ COMPATIBILITY="ALL"
|
||||
<DEFAULT>"."</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="61" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="61" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
|
||||
<DEFAULT>true</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
@@ -280,12 +280,12 @@ COMPATIBILITY="ALL"
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
|
||||
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@ if (sftp) { // *** sftp *** //
|
||||
}
|
||||
globalMap.put("<%=cid %>_DIRECTORYNAME", <%=directoryname %>);
|
||||
<%}
|
||||
} else { // *** ftp+s *** //
|
||||
} else if (ftps) { // *** ftps *** //
|
||||
%>
|
||||
//change working dir and save root
|
||||
String remoteDir_<%=cid %> = (<%=remoteDir %>).replaceAll("\\\\","/");
|
||||
@@ -155,5 +155,35 @@ if (sftp) { // *** sftp *** //
|
||||
ftp_<%=cid %>.changeWorkingDirectory(remoteDir_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
} else { // *** ftp *** //
|
||||
if (isFileType){%>
|
||||
String remoteDir_<%=cid %> = (<%=remoteDir %>).replaceAll("\\\\","/");
|
||||
if (ftp_<%=cid %>.exists<%="File" %>(remoteDir_<%=cid %> + "/" + <%=filename %>)) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - '" + <%=filename%> + "' exists in the remote directory '"+ <%=remoteDir %> + "' .");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_EXISTS", true);
|
||||
} else {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - '" + <%=filename%> + "' doesn't exist in the remote directory '"+ <%=remoteDir %> + "' .");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_EXISTS", false);
|
||||
}
|
||||
globalMap.put("<%=cid %>_FILENAME", <%=filename %>);
|
||||
<%} else {%>
|
||||
String remoteDir_<%=cid %> = (<%=remoteDir %>).replaceAll("\\\\","/");
|
||||
String directoryname_<%=cid %> = (<%=directoryname %>).replaceAll("\\\\","/");
|
||||
if (ftp_<%=cid %>.exists<%="Directory" %>(remoteDir_<%=cid %> + "/" + directoryname_<%=cid %>)) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - '" + <%=directoryname %> + "' exists in the remote directory '"+ <%=remoteDir %> + "' .");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_EXISTS", true);
|
||||
} else {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - '" + <%=directoryname %> + "' doesn't exist in the remote directory '"+ <%=remoteDir %> + "' .");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_EXISTS", false);
|
||||
}
|
||||
globalMap.put("<%=cid %>_DIRECTORYNAME", <%=directoryname %>);
|
||||
<%}
|
||||
}%>
|
||||
@@ -56,8 +56,6 @@ if (("true").equals(useExistingConn)) {
|
||||
} else {
|
||||
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
|
||||
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
|
||||
%>
|
||||
<%
|
||||
}
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
String passwordFieldName = "";
|
||||
@@ -153,7 +151,7 @@ if (sftp) {// *** sftp *** //
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
|
||||
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
|
||||
|
||||
<%if(use_encoding) {%>
|
||||
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
|
||||
@@ -234,104 +232,255 @@ if (sftp) {// *** sftp *** //
|
||||
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%>);
|
||||
nb_file_<%=cid%>++;
|
||||
<%
|
||||
} else {
|
||||
if (ftps) { // *** ftps *** //
|
||||
} else if (ftps) { // *** ftps *** //
|
||||
if (("false").equals(useExistingConn)) {
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
|
||||
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
|
||||
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
|
||||
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
|
||||
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
|
||||
|
||||
try {
|
||||
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
|
||||
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
|
||||
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
|
||||
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
|
||||
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
|
||||
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
|
||||
if (isHTTPProxyUsed) {
|
||||
String httpProxyHost = System.getProperty("http.proxyHost");
|
||||
int httpProxyPort = Integer.getInteger("http.proxyPort");
|
||||
String httpProxyUser = System.getProperty("http.proxyUser");
|
||||
String httpProxyPass = System.getProperty("http.proxyPassword");
|
||||
|
||||
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
|
||||
} else {
|
||||
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
|
||||
}
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
|
||||
<%
|
||||
if (("false").equals(useExistingConn)) {
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%
|
||||
} else {
|
||||
}
|
||||
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
|
||||
if (!isLoginSuccessful_<%=cid %>) {
|
||||
throw new RuntimeException("Login failed");
|
||||
}
|
||||
|
||||
<%
|
||||
}
|
||||
} else { // *** ftp
|
||||
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
|
||||
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
|
||||
<%
|
||||
if (("true").equals(useExistingConn)) {%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%
|
||||
} else {
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
}
|
||||
|
||||
if (protectionLevel != null && !protectionLevel.isEmpty()) {
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
|
||||
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
|
||||
<%
|
||||
}
|
||||
} //common code for ftp and ftps:
|
||||
}
|
||||
%>
|
||||
} catch (Exception e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
|
||||
<%}%>
|
||||
throw e;
|
||||
}
|
||||
<%
|
||||
if ("PASSIVE".equals(connectMode)) {
|
||||
%>
|
||||
ftp_<%=cid %>.enterLocalPassiveMode();
|
||||
<%
|
||||
if (isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - Using the passive mode.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
String rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
|
||||
<%
|
||||
}
|
||||
%>
|
||||
int nb_file_<%=cid%> = 0;
|
||||
org.apache.commons.net.ftp.FTPFile[] ftpFiles_<%=cid %> = null;
|
||||
List<org.apache.commons.net.ftp.FTPFile> fileListTemp_<%=cid %> = new java.util.ArrayList<>();
|
||||
|
||||
<%/*read files*/ %>
|
||||
String remotedir_<%=cid %> = (<%=remotedir%>).replaceAll("\\\\","/");
|
||||
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
|
||||
|
||||
if (!cwdSuccess_<%=cid %>) {
|
||||
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
|
||||
}
|
||||
|
||||
ftpFiles_<%=cid %> = ftp_<%=cid %>.listFiles();
|
||||
<%
|
||||
if ("true".equals(useExistingConn) && !moveToCurrentDir) {
|
||||
%>
|
||||
<%/*return to previous dir */%>
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%/*prepare masks*/ %>
|
||||
for (String filemask_<%=cid %> : maskList_<%=cid %>) {
|
||||
java.util.regex.Pattern fileNamePattern_<%=cid %> = java.util.regex.Pattern.compile(filemask_<%=cid %>.replaceAll("\\.", "\\\\.").replaceAll("\\*", ".*"));
|
||||
|
||||
for (org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> : ftpFiles_<%=cid %>) {
|
||||
if (fileNamePattern_<%=cid %>.matcher(ftpFile_<%=cid %>.getName()).matches()) {
|
||||
fileListTemp_<%=cid %>.add(ftpFile_<%=cid %>);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String currentFilePath_<%=cid%> = remotedir_<%=cid %>;
|
||||
if(!remotedir_<%=cid%>.endsWith("/")&&!remotedir_<%=cid%>.endsWith("\\")){
|
||||
currentFilePath_<%=cid%> += "/";
|
||||
}
|
||||
|
||||
for (int i = 0; i < fileListTemp_<%=cid %>.size(); i++) {
|
||||
org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> = fileListTemp_<%=cid %>.get(i);
|
||||
String currentFileName_<%=cid%> = null;
|
||||
<%
|
||||
if (dirFull) {
|
||||
%>
|
||||
currentFileName_<%=cid%> = ftpFile_<%=cid %>.toString();
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
currentFileName_<%=cid%> = ftpFile_<%=cid %>.getName();
|
||||
<%
|
||||
}
|
||||
|
||||
if(isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - List file : '" + currentFilePath_<%=cid%> + "' .");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
int nb_file_<%=cid%> = 0;
|
||||
org.apache.commons.net.ftp.FTPFile[] ftpFiles_<%=cid %> = null;
|
||||
String rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
|
||||
List<org.apache.commons.net.ftp.FTPFile> fileListTemp_<%=cid %> = new java.util.ArrayList<>();
|
||||
|
||||
<%/*read files*/ %>
|
||||
String remotedir_<%=cid %> = (<%=remotedir%>).replaceAll("\\\\","/");
|
||||
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
|
||||
|
||||
if (!cwdSuccess_<%=cid %>) {
|
||||
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
|
||||
}
|
||||
|
||||
ftpFiles_<%=cid %> = ftp_<%=cid %>.listFiles();
|
||||
<%/*set to globalMap*/ %>
|
||||
globalMap.put("<%=cid %>_CURRENT_FILE", currentFileName_<%=cid%>);
|
||||
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%> + ftpFile_<%=cid %>.getName());
|
||||
nb_file_<%=cid %>++;
|
||||
<%
|
||||
if ("true".equals(useExistingConn) && !moveToCurrentDir) {
|
||||
} else { // *** ftp *** //
|
||||
%>
|
||||
<%/*return to previous dir */%>
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%/*prepare masks*/ %>
|
||||
for (String filemask_<%=cid %> : maskList_<%=cid %>) {
|
||||
java.util.regex.Pattern fileNamePattern_<%=cid %> = java.util.regex.Pattern.compile(filemask_<%=cid %>.replaceAll("\\.", "\\\\.").replaceAll("\\*", ".*"));
|
||||
int nb_file_<%=cid%> = 0;
|
||||
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> =null;
|
||||
|
||||
for (org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> : ftpFiles_<%=cid %>) {
|
||||
if (fileNamePattern_<%=cid %>.matcher(ftpFile_<%=cid %>.getName()).matches()) {
|
||||
fileListTemp_<%=cid %>.add(ftpFile_<%=cid %>);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String currentFilePath_<%=cid%> = remotedir_<%=cid %>;
|
||||
if(!remotedir_<%=cid%>.endsWith("/")&&!remotedir_<%=cid%>.endsWith("\\")){
|
||||
currentFilePath_<%=cid%> += "/";
|
||||
}
|
||||
|
||||
for (int i = 0; i < fileListTemp_<%=cid %>.size(); i++) {
|
||||
org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> = fileListTemp_<%=cid %>.get(i);
|
||||
String currentFileName_<%=cid%> = null;
|
||||
<%if (("true").equals(useExistingConn)) {%>
|
||||
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%if(!moveToCurrentDir){%>
|
||||
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
|
||||
<%}%>
|
||||
<%} else {%>
|
||||
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
|
||||
ftp_<%=cid %>.setRemoteHost(<%=host %>);
|
||||
ftp_<%=cid %>.setRemotePort(<%=port %>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
|
||||
<%}%>
|
||||
<%if (("ACTIVE").equals(connectMode)) {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
|
||||
<%} else {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
|
||||
<%}%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
ftp_<%=cid %>.connect();
|
||||
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
<%}%>
|
||||
String remotedir_<%=cid %> = (<%=remotedir%>).replaceAll("\\\\","/");
|
||||
ftp_<%=cid %>.chdir(remotedir_<%=cid %>);
|
||||
String[] fileList_<%=cid %> = ftp_<%=cid %>.dir(null, true);
|
||||
String[] fileListwithNames_<%=cid %> = ftp_<%=cid %>.dir(null, false);
|
||||
<%
|
||||
if (dirFull) {
|
||||
if (("true").equals(useExistingConn)) {
|
||||
if(!moveToCurrentDir){
|
||||
%>
|
||||
currentFileName_<%=cid%> = ftpFile_<%=cid %>.toString();
|
||||
<%
|
||||
} else {
|
||||
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
currentFileName_<%=cid%> = ftpFile_<%=cid %>.getName();
|
||||
<%
|
||||
}
|
||||
List<String> fileListTemp_<%=cid %> = new java.util.ArrayList<String>();
|
||||
List<String> fileListTempWithNames_<%=cid %> = new java.util.ArrayList<String>();
|
||||
|
||||
if(isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - List file : '" + currentFilePath_<%=cid%> + "' .");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%/*set to globalMap*/ %>
|
||||
globalMap.put("<%=cid %>_CURRENT_FILE", currentFileName_<%=cid%>);
|
||||
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%> + ftpFile_<%=cid %>.getName());
|
||||
nb_file_<%=cid %>++;
|
||||
<%
|
||||
} //end common code part
|
||||
%>
|
||||
for (String filemask_<%=cid %> : maskList_<%=cid %>) {
|
||||
java.util.regex.Pattern fileNamePattern_<%=cid %> = java.util.regex.Pattern.compile(filemask_<%=cid %>.replaceAll("\\.", "\\\\.").replaceAll("\\*", ".*"));
|
||||
for (int i_<%=cid%> = 0; i_<%=cid%> < fileList_<%=cid%>.length; i_<%=cid%>++){
|
||||
String filemaskTemp_<%=cid%> = fileList_<%=cid%>[i_<%=cid%>];
|
||||
String filemaskTempName_<%=cid%> = fileListwithNames_<%=cid%>[i_<%=cid%>];
|
||||
|
||||
if (fileNamePattern_<%=cid %>.matcher(filemaskTempName_<%=cid %>).matches()) {
|
||||
fileListTemp_<%=cid %>.add(filemaskTemp_<%=cid %>);
|
||||
fileListTempWithNames_<%=cid%>.add(fileListwithNames_<%=cid%>[i_<%=cid%>]);
|
||||
}
|
||||
}
|
||||
}
|
||||
int i_<%=cid %> = -1;
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Listing files from server.");
|
||||
<%}%>
|
||||
while (++i_<%=cid %> < fileListTemp_<%=cid %>.size()) {
|
||||
<% if (dirFull){%>
|
||||
String currentFileName_<%=cid%> = fileListTemp_<%=cid %>.get(i_<%=cid %>);
|
||||
<% } else {%>
|
||||
String currentFileName_<%=cid%> = fileListTempWithNames_<%=cid %>.get(i_<%=cid %>);
|
||||
<% }%>
|
||||
|
||||
String currentFilePath_<%=cid%> = remotedir_<%=cid %>;
|
||||
if(!remotedir_<%=cid%>.endsWith("/")&&!remotedir_<%=cid%>.endsWith("\\")){
|
||||
currentFilePath_<%=cid%> += "/";
|
||||
}
|
||||
currentFilePath_<%=cid%> += fileListTempWithNames_<%=cid%>.get(i_<%=cid%>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - List file : '" + currentFilePath_<%=cid%> + "' .");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_FILE", currentFileName_<%=cid%>);
|
||||
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%>);
|
||||
nb_file_<%=cid%>++;
|
||||
<%}%>
|
||||
|
||||
@@ -65,14 +65,14 @@ if (sftp && !("true").equals(useExistingConn)) {// *** sftp *** //
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
} catch (java.io.IOException ioe_<%=cid%>) {
|
||||
} catch (java.net.SocketException se_<%=cid%>) {
|
||||
//ignore failure
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
}
|
||||
<%
|
||||
@@ -81,7 +81,7 @@ if (sftp && !("true").equals(useExistingConn)) {// *** sftp *** //
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
|
||||
@@ -155,7 +155,7 @@
|
||||
<DEFAULT>"."</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="41" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="41" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
|
||||
<DEFAULT>true</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
@@ -258,12 +258,12 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
|
||||
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS>
|
||||
|
||||
@@ -62,8 +62,6 @@ if (("true").equals(useExistingConn)) {
|
||||
} else {
|
||||
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
|
||||
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
|
||||
%>
|
||||
<%
|
||||
}
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
@@ -155,9 +153,9 @@ if (sftp) { // *** sftp *** //
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
|
||||
|
||||
|
||||
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
|
||||
|
||||
<%if(use_encoding) {%>
|
||||
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
|
||||
<%}%>
|
||||
@@ -204,52 +202,93 @@ if (sftp) { // *** sftp *** //
|
||||
<%}%>
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
if (ftps) { // *** ftps *** //
|
||||
} else if (ftps) { // *** ftps *** //
|
||||
if (("false").equals(useExistingConn)) {
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
<%
|
||||
if (("false").equals(useExistingConn)) {
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
<%
|
||||
}
|
||||
} else { // *** ftp *** //
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
|
||||
<%
|
||||
if (("true").equals(useExistingConn)) {
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%
|
||||
} else {
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
|
||||
<%
|
||||
}
|
||||
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
|
||||
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
|
||||
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
|
||||
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
|
||||
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
|
||||
try {
|
||||
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
|
||||
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
|
||||
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
|
||||
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
|
||||
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
|
||||
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
|
||||
if (isHTTPProxyUsed) {
|
||||
String httpProxyHost = System.getProperty("http.proxyHost");
|
||||
int httpProxyPort = Integer.getInteger("http.proxyPort");
|
||||
String httpProxyUser = System.getProperty("http.proxyUser");
|
||||
String httpProxyPass = System.getProperty("http.proxyPassword");
|
||||
|
||||
if("binary".equalsIgnoreCase(mode)){
|
||||
%>
|
||||
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
|
||||
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
|
||||
} else {
|
||||
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
|
||||
}
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
|
||||
<%
|
||||
} else {
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.ASCII_FILE_TYPE);
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
|
||||
if (!isLoginSuccessful_<%=cid %>) {
|
||||
throw new RuntimeException("Login failed");
|
||||
}
|
||||
|
||||
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE); //to calculate md5 for binary files correctly
|
||||
<%
|
||||
}
|
||||
} //common code for ftp and ftps:
|
||||
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
|
||||
<%
|
||||
}
|
||||
|
||||
if (protectionLevel != null && !protectionLevel.isEmpty()) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} catch (Exception e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
|
||||
<%}%>
|
||||
throw e;
|
||||
}
|
||||
<%
|
||||
if ("PASSIVE".equals(connectMode)) {
|
||||
%>
|
||||
ftp_<%=cid %>.enterLocalPassiveMode();
|
||||
<%
|
||||
if (isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - Using the passive mode.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
String remoteDir_<%=cid %> = (<%=remoteDir %>).replaceAll("\\\\", "/");
|
||||
org.apache.commons.net.ftp.FTPFile[] allFiles_<%=cid %> = ftp_<%=cid %>.listFiles(remoteDir_<%=cid %>);
|
||||
@@ -291,5 +330,84 @@ if (sftp) { // *** sftp *** //
|
||||
<%}%>
|
||||
}
|
||||
<%
|
||||
} else {// *** ftp *** //
|
||||
%>
|
||||
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> =null;
|
||||
|
||||
<%if (("true").equals(useExistingConn)) {%>
|
||||
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%} else {%>
|
||||
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
|
||||
ftp_<%=cid %>.setRemoteHost(<%=host %>);
|
||||
ftp_<%=cid %>.setRemotePort(<%=port %>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
|
||||
<%}%>
|
||||
<%if (("ACTIVE").equals(connectMode)) {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
|
||||
<%} else {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect();
|
||||
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
ftp_<%=cid %>.login(<%=username %>, decryptedPassword_<%=cid%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
<%}%>
|
||||
<%
|
||||
//add feature 19709,add the "Transfer mode" option,
|
||||
//the default transfer mode is ASCII,we don't change the mode when ascii is choosed.
|
||||
if("binary".equalsIgnoreCase(mode)){
|
||||
%>
|
||||
ftp_<%=cid %>.setType(com.enterprisedt.net.ftp.FTPTransferType.BINARY);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
String remoteDir_<%=cid %> = <%=remoteDir %>.replaceAll("\\\\", "/");
|
||||
<%=outputConnName %> = new <%=outputConnName %>Struct();
|
||||
String remoteFile_<%=cid %> = <%=remoteDir %> + "/" + <%=filename%>;
|
||||
|
||||
if (ftp_<%=cid %>.exists(remoteFile_<%=cid %>)) {
|
||||
<%=outputConnName %>.abs_path = (remoteFile_<%=cid %>).replaceAll("//", "/");
|
||||
<%=outputConnName %>.dirname = <%=remoteDir %>;
|
||||
<%=outputConnName %>.basename = <%=filename%>;
|
||||
<%=outputConnName %>.size = ftp_<%=cid %>.size(remoteFile_<%=cid %>);
|
||||
<%=outputConnName %>.mtime = ftp_<%=cid %>.modtime(remoteFile_<%=cid %>).getTime();
|
||||
<%=outputConnName %>.mtime_string =ftp_<%=cid %>.modtime(remoteFile_<%=cid %>).toString();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log4jSb_<%=cid%>.append("<%=cid%> - File properties for remoteDir is : '" + <%=remoteDir %> + "',file name is : '" + <%=filename%> +"',file size is : '" + <%=outputConnName %>.size + "',file last modify time is : '" + <%=outputConnName %>.mtime_string + "'");
|
||||
<%}%>
|
||||
|
||||
<%if (MD5) {%>
|
||||
// Calculation of the Message Digest MD5
|
||||
java.security.MessageDigest dgs_<%=cid %> = java.security.MessageDigest.getInstance("MD5");
|
||||
dgs_<%=cid %>.update(ftp_<%=cid %>.get(remoteFile_<%=cid %>));
|
||||
<%=outputConnName %>.md5 =String.format("%032x", new java.math.BigInteger(1, dgs_<%=cid %>.digest()));
|
||||
<%if(isLog4jEnabled){%>
|
||||
log4jSb_<%=cid%>.append(",md5 message is : '"+ <%=outputConnName %>.md5 + "'.");
|
||||
<%}%>
|
||||
<%}%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug(log4jSb_<%=cid%>);
|
||||
<%}%>
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -80,13 +80,13 @@ if (sftp && !bUseExistingConn) { // *** sftp *** //
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
} catch (java.io.IOException ioe_<%=cid%>) {
|
||||
} catch (java.net.SocketException se_<%=cid%>) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
//ignore failure
|
||||
}
|
||||
@@ -94,7 +94,7 @@ if (sftp && !bUseExistingConn) { // *** sftp *** //
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
|
||||
@@ -255,12 +255,12 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
|
||||
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS />
|
||||
|
||||
@@ -66,8 +66,6 @@ if ("true".equals(useExistingConn)) {
|
||||
} else {
|
||||
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
|
||||
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
|
||||
%>
|
||||
<%
|
||||
}
|
||||
%>
|
||||
int nb_file_<%=cid%> = 0;
|
||||
@@ -288,7 +286,7 @@ if (sftp) { // *** sftp *** //
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
|
||||
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
|
||||
|
||||
<%if(use_encoding) {%>
|
||||
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
|
||||
@@ -305,247 +303,477 @@ if (sftp) { // *** sftp *** //
|
||||
c_<%=cid%>.cd(remotedir_<%=cid %>);
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
abstract class FTPGetter_<%=cid%> {
|
||||
protected org.apache.commons.net.ftp.FTPClient ftpClient = null;
|
||||
protected int count = 0;
|
||||
} else if (!ftps) { // *** ftp *** //
|
||||
%>
|
||||
final com.enterprisedt.net.ftp.TransferCompleteStrings msg_<%=cid%> = new com.enterprisedt.net.ftp.TransferCompleteStrings();
|
||||
|
||||
public void getAllFiles(String remoteDirectory, String localDirectory)
|
||||
throws IllegalStateException, IOException, java.io.FileNotFoundException {
|
||||
class FTPGetter_<%=cid%> {
|
||||
private com.enterprisedt.net.ftp.FTPClient ftpClient = null;
|
||||
private int count = 0;
|
||||
|
||||
chdir(remoteDirectory);
|
||||
String path = ftpClient.printWorkingDirectory();
|
||||
org.apache.commons.net.ftp.FTPFile[] ftpFiles = null;
|
||||
ftpFiles = ftpClient.listFiles();
|
||||
public void getAllFiles(String remoteDirectory, String localDirectory) throws java.io.IOException,
|
||||
com.enterprisedt.net.ftp.FTPException, java.text.ParseException {
|
||||
|
||||
for (org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
|
||||
chdir(remoteDirectory);
|
||||
String path = ftpClient.pwd();
|
||||
String[] ftpFileNames = null;
|
||||
com.enterprisedt.net.ftp.FTPFile[] ftpFiles = null;
|
||||
|
||||
if (ftpFile.isDirectory()) {
|
||||
try{
|
||||
//use dir() for Bug9562 with FTP server in AS400
|
||||
ftpFileNames = ftpClient.dir(null, false);
|
||||
//Bug 13272, the same as getFiles().
|
||||
} catch (com.enterprisedt.net.ftp.FTPException e){
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + e.getMessage());
|
||||
<%}%>
|
||||
ftpFileNames = null;
|
||||
ftpFiles = ftpClient.dirDetails(".");
|
||||
}
|
||||
|
||||
if ((!(".").equals(ftpFile.getName())) && (!("..").equals(ftpFile.getName()))) {
|
||||
java.io.File localFile = new java.io.File(localDirectory + "/" + ftpFile.getName());
|
||||
//if dirDetails(...) doesn't work, then use dir(...), distinguish file type by FTPException
|
||||
|
||||
if (!localFile.exists()) {
|
||||
localFile.mkdir();
|
||||
}
|
||||
getAllFiles(path + "/" + ftpFile.getName(), localDirectory + "/" + ftpFile.getName());
|
||||
chdir(path);
|
||||
}
|
||||
} else if (!ftpFile.isSymbolicLink()) {
|
||||
downloadFile(localDirectory + "/" + ftpFile.getName(),ftpFile.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
if ((ftpFiles == null) && (ftpFileNames != null)){
|
||||
//if the file is folder, catch the FTPException and recur
|
||||
for (String ftpFileName : ftpFileNames){
|
||||
try {
|
||||
downloadFile(localDirectory + "/" + ftpFileName,ftpFileName);
|
||||
} catch (com.enterprisedt.net.ftp.FTPException e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + e.getMessage());
|
||||
<%}%>
|
||||
java.io.File localFile = new java.io.File(localDirectory + "/" + ftpFileName);
|
||||
|
||||
public void getFiles(String remoteDirectory, String localDirectory, String maskStr)
|
||||
throws IllegalStateException, IOException, java.io.FileNotFoundException {
|
||||
if (!localFile.exists()) {
|
||||
localFile.mkdir();
|
||||
}
|
||||
getAllFiles(path + "/" + ftpFileName, localDirectory + "/" + ftpFileName);
|
||||
chdir(path);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (com.enterprisedt.net.ftp.FTPFile ftpFile : ftpFiles) {
|
||||
|
||||
chdir(remoteDirectory);
|
||||
org.apache.commons.net.ftp.FTPFile[] ftpFiles = null;
|
||||
ftpFiles = ftpClient.listFiles(".");
|
||||
if (ftpFile.isDir()) {
|
||||
if ((!(".").equals(ftpFile.getName())) && (!("..").equals(ftpFile.getName()))) {
|
||||
java.io.File localFile = new java.io.File(localDirectory + "/" + ftpFile.getName());
|
||||
|
||||
for(org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
|
||||
if (!localFile.exists()) {
|
||||
localFile.mkdir();
|
||||
}
|
||||
getAllFiles(path + "/" + ftpFile.getName(), localDirectory + "/" + ftpFile.getName());
|
||||
chdir(path);
|
||||
}
|
||||
} else if (!ftpFile.isLink()) {
|
||||
downloadFile(localDirectory + "/" + ftpFile.getName(),ftpFile.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ftpFile.isFile()) {
|
||||
String fileName = ftpFile.getName();
|
||||
public void getFiles(String remoteDirectory, String localDirectory, String maskStr) throws java.io.IOException,
|
||||
com.enterprisedt.net.ftp.FTPException, java.text.ParseException {
|
||||
chdir(remoteDirectory);
|
||||
String[] ftpFileNames = null;
|
||||
com.enterprisedt.net.ftp.FTPFile[] ftpFiles = null;
|
||||
|
||||
if (fileName.matches(maskStr)) {
|
||||
downloadFile(localDirectory + "/" + fileName,fileName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void chdir(String path) throws IllegalStateException, IOException {
|
||||
if (!".".equals(path)) {
|
||||
ftpClient.changeWorkingDirectory(path);
|
||||
}
|
||||
}
|
||||
|
||||
public String pwd() throws IllegalStateException, IOException {
|
||||
return ftpClient.printWorkingDirectory();
|
||||
}
|
||||
|
||||
protected abstract void downloadFile(String localFileName, String remoteFileName)
|
||||
throws IllegalStateException, java.io.FileNotFoundException, IOException;
|
||||
}
|
||||
<%
|
||||
if (!ftps) { // *** ftp *** //
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
|
||||
<%
|
||||
if (("true").equals(useExistingConn)) {
|
||||
String conn= "conn_" + connection;
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
|
||||
<%if(!moveToCurrentDir){%>
|
||||
String rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
|
||||
<%}%>
|
||||
try {
|
||||
//use dir() for Bug9562 with FTP server in AS400 (the same way as getAllFiles())
|
||||
ftpFileNames = ftpClient.dir(null, false);
|
||||
//Bug 13272, if dir() throw exception, use dirDetails().
|
||||
} catch (com.enterprisedt.net.ftp.FTPException e){
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%
|
||||
} else {
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
|
||||
<%
|
||||
}
|
||||
log.warn("<%=cid%> - " + e.getMessage());
|
||||
<%}%>
|
||||
ftpFileNames = null;
|
||||
ftpFiles = ftpClient.dirDetails(".");
|
||||
}
|
||||
//if dirDetails(...) doesn't work, then use dir(...), but can not distinguish file type
|
||||
|
||||
if("binary".equalsIgnoreCase(ElementParameterParser.getValue(node, "__MODE__"))){
|
||||
%>
|
||||
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.ASCII_FILE_TYPE);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
final java.util.List<String> msg_<%=cid%> = new java.util.ArrayList<String>();
|
||||
FTPGetter_<%=cid%> getter_<%=cid%> = new FTPGetter_<%=cid%>() {
|
||||
@Override
|
||||
protected void downloadFile(String localFileName, String remoteFileName)
|
||||
throws IllegalStateException, java.io.FileNotFoundException, IOException {
|
||||
java.io.File localFile = new java.io.File(localFileName);
|
||||
<%
|
||||
if (append) {
|
||||
%>
|
||||
try (java.io.FileOutputStream localFos = new java.io.FileOutputStream(localFileName, true)) {
|
||||
ftpClient.retrieveFile(remoteFileName, localFos);
|
||||
if ((ftpFiles == null) && (ftpFileNames != null)){
|
||||
for (String ftpFileName : ftpFileNames) {
|
||||
if (ftpFileName.matches(maskStr)) {
|
||||
downloadFile(localDirectory + "/" + ftpFileName,ftpFileName);
|
||||
}
|
||||
}
|
||||
} else{
|
||||
|
||||
for (com.enterprisedt.net.ftp.FTPFile ftpFile : ftpFiles) {
|
||||
|
||||
if (!ftpFile.isDir() && !ftpFile.isLink()) {
|
||||
String fileName = ftpFile.getName();
|
||||
|
||||
if (fileName.matches(maskStr)) {
|
||||
downloadFile(localDirectory + "/" + fileName,fileName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void chdir(String path) throws java.io.IOException, com.enterprisedt.net.ftp.FTPException{
|
||||
if (!".".equals(path)) {
|
||||
ftpClient.chdir(path);
|
||||
}
|
||||
}
|
||||
|
||||
public String pwd() throws java.io.IOException, com.enterprisedt.net.ftp.FTPException{
|
||||
return ftpClient.pwd();
|
||||
}
|
||||
|
||||
private void downloadFile(String localFileName, String remoteFileName) throws java.io.IOException, com.enterprisedt.net.ftp.FTPException {
|
||||
java.io.File localFile = new java.io.File(localFileName);
|
||||
|
||||
try {
|
||||
<%if (append) {%>
|
||||
java.io.FileOutputStream fout = new java.io.FileOutputStream(localFile, true);
|
||||
ftpClient.get(fout, remoteFileName);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
|
||||
<%}%>
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%if (bPrintMsg) {%>
|
||||
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
|
||||
<%
|
||||
} else if ("never".equals(overwrite)){
|
||||
%>
|
||||
if (!localFile.exists()) {
|
||||
ftpClient.get(localFileName, remoteFileName);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
|
||||
<%}%>
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%if (bPrintMsg) {%>
|
||||
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
|
||||
} else {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
|
||||
log.info("<%= cid %> - file ["+ remoteFileName +"] exit transmission.");
|
||||
<%}%>
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%if (bPrintMsg) {%>
|
||||
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
|
||||
count++;
|
||||
} catch (IOException e) {
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded unsuccessfully.");
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer fail.");
|
||||
throw e;
|
||||
}
|
||||
<%
|
||||
} else if ("never".equals(overwrite)) {
|
||||
%>
|
||||
if (!localFile.exists()) {
|
||||
downloadFileWithOverwrite(localFileName, remoteFileName);
|
||||
} else {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%= cid %> - file ["+ remoteFileName +"] exit transmission.");
|
||||
<%}%>
|
||||
msg_<%=cid%>.add("file ["+ remoteFileName +"] exit transmission.");
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file transfered.");
|
||||
}
|
||||
<%
|
||||
} else if ("always".equals(overwrite)){
|
||||
%>
|
||||
downloadFileWithOverwrite(localFileName, remoteFileName);
|
||||
<%
|
||||
} else if ("size_differ".equals(overwrite)) {
|
||||
%>
|
||||
if (localFile.exists()) {
|
||||
long ftpSize = java.util.Arrays.stream(ftpClient.listFiles(remoteFileName)).findFirst().get().getSize();
|
||||
long localSize = localFile.length();
|
||||
if (ftpSize != localSize) {
|
||||
downloadFileWithOverwrite(localFileName, remoteFileName);
|
||||
} else {
|
||||
<%if(isLog4jEnabled){%>
|
||||
msg_<%=cid%>.add("file ["+ remoteFileName +"] exit transmission.");
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file transfered.");
|
||||
}
|
||||
<%
|
||||
} else if ("always".equals(overwrite)){
|
||||
%>
|
||||
ftpClient.get(localFileName, remoteFileName);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
|
||||
<%}%>
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%if (bPrintMsg) {%>
|
||||
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%}%>
|
||||
<%
|
||||
} else if ("size_differ".equals(overwrite)) {
|
||||
%>
|
||||
if (localFile.exists()) {
|
||||
long ftpSize = ftpClient.size(remoteFileName);
|
||||
long localSize = localFile.length();
|
||||
|
||||
if (ftpSize != localSize) {
|
||||
ftpClient.get(localFileName, remoteFileName);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
|
||||
<%}%>
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%if (bPrintMsg) {%>
|
||||
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
|
||||
} else {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%= cid %> - file ["+ remoteFileName +"] exit transmission.");
|
||||
<%}%>
|
||||
msg_<%=cid%>.add("file ["+ remoteFileName +"] exit transmission.");
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file transfered.");
|
||||
}
|
||||
} else {
|
||||
downloadFileWithOverwrite(localFileName, remoteFileName);
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
|
||||
private void downloadFileWithOverwrite(String localFileName, String remoteFileName)
|
||||
throws IllegalStateException, java.io.FileNotFoundException, IOException {
|
||||
try (java.io.FileOutputStream localFos = new java.io.FileOutputStream(localFileName)) {
|
||||
ftpClient.retrieveFile(remoteFileName, localFos);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
|
||||
<%}%>
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%if (bPrintMsg) {%>
|
||||
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
|
||||
count++;
|
||||
} catch (IOException e) {
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded unsuccessfully.");
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer fail.");
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
};
|
||||
getter_<%=cid%>.ftpClient = ftp_<%=cid %>;
|
||||
<%
|
||||
} else { // *** ftps *** //
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
|
||||
<%
|
||||
if ("false".equals(useExistingConn)){
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
|
||||
<%
|
||||
} else {
|
||||
String conn= "conn_" + connection;
|
||||
%>
|
||||
<% /* get connection from globalMap*/ %>
|
||||
String rootDir_<%=cid %> = null;
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient)globalMap.get("<%=conn %>");
|
||||
<%
|
||||
if (!moveToCurrentDir) {
|
||||
%>
|
||||
rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
|
||||
<%
|
||||
}
|
||||
|
||||
if(isLog4jEnabled) {
|
||||
%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection.Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
FTPGetter_<%=cid%> getter_<%=cid%> = new FTPGetter_<%=cid%>() {
|
||||
@Override
|
||||
protected void downloadFile(String localFileName, String remoteFileName)
|
||||
throws IllegalStateException, java.io.FileNotFoundException, IOException {
|
||||
try (java.io.FileOutputStream localFos = new java.io.FileOutputStream(localFileName)) {
|
||||
ftpClient.retrieveFile(remoteFileName, localFos);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
|
||||
<%}%>
|
||||
count++;
|
||||
}
|
||||
}
|
||||
};
|
||||
getter_<%=cid%>.ftpClient = ftp_<%=cid %>;
|
||||
<%
|
||||
} //common code for ftp and ftps:
|
||||
%>
|
||||
String remotedir_<%=cid%> = <%=remotedir%>;
|
||||
if (!".".equals(remotedir_<%=cid %>)) {
|
||||
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
|
||||
|
||||
if (!cwdSuccess_<%=cid %>) {
|
||||
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
|
||||
}
|
||||
msg_<%=cid%>.add("file ["+ remoteFileName +"] exit transmission.");
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file transfered.");
|
||||
}
|
||||
} else {
|
||||
ftpClient.get(localFileName, remoteFileName);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
|
||||
<%}%>
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%if (bPrintMsg) {%>
|
||||
System.out.println("file [" + remoteFileName + "] downloaded successfully.");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} catch (com.enterprisedt.net.ftp.FTPException e) {
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded unsuccessfully.");
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer fail.");
|
||||
throw e;
|
||||
}
|
||||
count++;
|
||||
}
|
||||
}
|
||||
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> = null;
|
||||
|
||||
<%
|
||||
if ("true".equals(useExistingConn)){
|
||||
String conn= "conn_" + connection;
|
||||
%>
|
||||
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
|
||||
<%if(!moveToCurrentDir){%>
|
||||
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
|
||||
<%}%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Uses an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%} else {%>
|
||||
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
|
||||
ftp_<%=cid %>.setRemoteHost(<%=host %>);
|
||||
ftp_<%=cid %>.setRemotePort(<%=port %>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
|
||||
<%}
|
||||
if (("ACTIVE").equals(connectMode)){
|
||||
%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
|
||||
<%
|
||||
}else{
|
||||
%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect();
|
||||
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
<%} %>
|
||||
msg_<%=cid%>.clearAll();
|
||||
FTPGetter_<%=cid%> getter_<%=cid%> = new FTPGetter_<%=cid%>();
|
||||
getter_<%=cid%>.ftpClient = ftp_<%=cid %>;
|
||||
String remotedir_<%=cid %> = <%=remotedir%>;
|
||||
if (!".".equals(remotedir_<%=cid %>)) {
|
||||
ftp_<%=cid %>.chdir(remotedir_<%=cid %>);
|
||||
}
|
||||
<%
|
||||
} else { // *** ftps *** //
|
||||
%>
|
||||
class FTPSGetter_<%=cid%> {
|
||||
private org.apache.commons.net.ftp.FTPClient ftpClient = null;
|
||||
private int count = 0;
|
||||
|
||||
public void getAllFiles(String remoteDirectory, String localDirectory)
|
||||
throws IllegalStateException, IOException, java.io.FileNotFoundException {
|
||||
|
||||
chdir(remoteDirectory);
|
||||
String path = ftpClient.printWorkingDirectory();
|
||||
org.apache.commons.net.ftp.FTPFile[] ftpFiles = null;
|
||||
ftpFiles = ftpClient.listFiles();
|
||||
|
||||
for (org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
|
||||
|
||||
if (ftpFile.isDirectory()) {
|
||||
|
||||
if ((!(".").equals(ftpFile.getName())) && (!("..").equals(ftpFile.getName()))) {
|
||||
java.io.File localFile = new java.io.File(localDirectory + "/" + ftpFile.getName());
|
||||
|
||||
if (!localFile.exists()) {
|
||||
localFile.mkdir();
|
||||
}
|
||||
getAllFiles(path + "/" + ftpFile.getName(), localDirectory + "/" + ftpFile.getName());
|
||||
chdir(path);
|
||||
}
|
||||
} else if (!ftpFile.isSymbolicLink()) {
|
||||
downloadFile(localDirectory + "/" + ftpFile.getName(),ftpFile.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void getFiles(String remoteDirectory, String localDirectory, String maskStr)
|
||||
throws IllegalStateException, IOException, java.io.FileNotFoundException {
|
||||
|
||||
chdir(remoteDirectory);
|
||||
org.apache.commons.net.ftp.FTPFile[] ftpFiles = null;
|
||||
ftpFiles = ftpClient.listFiles(".");
|
||||
|
||||
for(org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
|
||||
|
||||
if (ftpFile.isFile()) {
|
||||
String fileName = ftpFile.getName();
|
||||
|
||||
if (fileName.matches(maskStr)) {
|
||||
downloadFile(localDirectory + "/" + fileName,fileName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void chdir(String path)
|
||||
throws IllegalStateException, IOException {
|
||||
if (!".".equals(path)) {
|
||||
ftpClient.changeWorkingDirectory(path);
|
||||
}
|
||||
}
|
||||
|
||||
public String pwd()
|
||||
throws IllegalStateException, IOException {
|
||||
return ftpClient.printWorkingDirectory();
|
||||
}
|
||||
|
||||
private void downloadFile(String localFileName, String remoteFileName)
|
||||
throws IllegalStateException, java.io.FileNotFoundException, IOException {
|
||||
try (java.io.FileOutputStream localFos = new java.io.FileOutputStream(localFileName)) {
|
||||
ftpClient.retrieveFile(remoteFileName, localFos);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
|
||||
<%}%>
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
|
||||
<%
|
||||
if ("false".equals(useExistingConn)){
|
||||
%>
|
||||
<% /* create connection */ %>
|
||||
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
|
||||
|
||||
javax.net.ssl.SSLContext sslContext_<%=cid%> = null;
|
||||
javax.net.ssl.TrustManager[] trustManager_<%=cid%> = null;
|
||||
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid%> = null;
|
||||
|
||||
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
|
||||
|
||||
try {
|
||||
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
|
||||
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
|
||||
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
|
||||
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
|
||||
sslSocketFactory_<%=cid %> = sslContext_<%=cid%>.getSocketFactory();
|
||||
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
|
||||
if (isHTTPProxyUsed) {
|
||||
String httpProxyHost = System.getProperty("http.proxyHost");
|
||||
int httpProxyPort = Integer.getInteger("http.proxyPort");
|
||||
String httpProxyUser = System.getProperty("http.proxyUser");
|
||||
String httpProxyPass = System.getProperty("http.proxyPassword");
|
||||
|
||||
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
|
||||
} else {
|
||||
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
|
||||
}
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
|
||||
if (!isLoginSuccessful_<%=cid %>) {
|
||||
throw new RuntimeException("Login failed");
|
||||
}
|
||||
|
||||
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
|
||||
<%
|
||||
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
|
||||
<%
|
||||
}
|
||||
|
||||
if (protectionLevel != null && !protectionLevel.isEmpty()) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} catch (Exception e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
|
||||
<%}%>
|
||||
<%
|
||||
if (dieOnError) {
|
||||
%>
|
||||
throw e;
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
e.printStackTrace();
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
<%
|
||||
if ("PASSIVE".equals(connectMode)) {
|
||||
%>
|
||||
ftp_<%=cid %>.enterLocalPassiveMode();
|
||||
<%
|
||||
if (isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - Using the passive mode.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
String conn= "conn_" + connection;
|
||||
%>
|
||||
<% /* get connection from globalMap*/ %>
|
||||
String rootDir_<%=cid %> = null;
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient)globalMap.get("<%=conn %>");
|
||||
<%
|
||||
if (!moveToCurrentDir) {
|
||||
%>
|
||||
rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
|
||||
<%
|
||||
}
|
||||
if(isLog4jEnabled) {
|
||||
%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection.Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
FTPSGetter_<%=cid%> getter_<%=cid%> = null;
|
||||
String remotedir_<%=cid%> = null;
|
||||
|
||||
getter_<%=cid%> = new FTPSGetter_<%=cid%>();
|
||||
getter_<%=cid%>.ftpClient = ftp_<%=cid %>;
|
||||
remotedir_<%=cid%> = <%=remotedir%>;
|
||||
if (!".".equals(remotedir_<%=cid %>)) {
|
||||
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
|
||||
|
||||
if (!cwdSuccess_<%=cid %>) {
|
||||
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
|
||||
}
|
||||
}
|
||||
<%
|
||||
}
|
||||
// *** share code *** //
|
||||
@@ -555,10 +783,22 @@ java.util.List<String> maskList_<%=cid %> = new java.util.ArrayList<String>();
|
||||
<%
|
||||
for (int i = 0; i < files.size(); i++) {
|
||||
Map<String, String> line = files.get(i);
|
||||
%>
|
||||
maskList_<%=cid %>.add(<%= line.get("FILEMASK") %>);
|
||||
%>
|
||||
maskList_<%=cid %>.add(<%= line.get("FILEMASK") %>);
|
||||
<%
|
||||
}
|
||||
|
||||
if (!sftp && !ftps) { // *** ftp *** //
|
||||
if ("binary".equalsIgnoreCase(ElementParameterParser.getValue(node, "__MODE__"))) {
|
||||
%>
|
||||
ftp_<%=cid %>.setType(com.enterprisedt.net.ftp.FTPTransferType.BINARY);
|
||||
<%
|
||||
}else {
|
||||
%>
|
||||
ftp_<%=cid %>.setType(com.enterprisedt.net.ftp.FTPTransferType.ASCII);
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
String localdir_<%=cid %> = <%=localdir%>;
|
||||
//create folder if local direcotry (assigned by property) not exists
|
||||
|
||||
@@ -54,8 +54,8 @@ nb_file_<%=cid%> = getter_<%=cid %>.count;
|
||||
<%}%>
|
||||
<%}%>
|
||||
<%} else if (!ftps) {%>
|
||||
msg_<%=cid%>.add(getter_<%=cid%>.count + " files have been downloaded.");
|
||||
String[] msgAll_<%=cid %> = msg_<%=cid%>.toArray(new String[0]);
|
||||
msg_<%=cid%>.add(ftp_<%=cid%>.getDownloadCount() + " files have been downloaded.");
|
||||
String[] msgAll_<%=cid %> = msg_<%=cid%>.getAll();
|
||||
StringBuffer sb_<%=cid %> = new StringBuffer();
|
||||
|
||||
if (msgAll_<%=cid %> != null) {
|
||||
@@ -74,13 +74,13 @@ nb_file_<%=cid%> = getter_<%=cid %>.count;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
} catch(java.io.IOException ioe_<%=cid%>) {
|
||||
} catch(java.net.SocketException se_<%=cid%>) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
log.error("<%=cid%> - " + se_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
//ignore failure
|
||||
}
|
||||
@@ -90,7 +90,7 @@ nb_file_<%=cid%> = getter_<%=cid %>.count;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
@@ -99,7 +99,7 @@ nb_file_<%=cid%> = getter_<%=cid %>.count;
|
||||
}else{
|
||||
if(!moveToCurrentDir){
|
||||
%>
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
|
||||
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,7 +150,7 @@ COMPATIBILITY="ALL"
|
||||
<DEFAULT>"."</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="61" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="61" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
|
||||
<DEFAULT>true</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
@@ -298,15 +298,15 @@ COMPATIBILITY="ALL"
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="jakarta-oro" MODULE="jakarta-oro-2.0.8.jar" MVN="mvn:org.talend.libraries/jakarta-oro-2.0.8/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache/lib/jakarta-oro-2.0.8.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
|
||||
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
|
||||
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
</CODEGENERATION>
|
||||
<RETURNS>
|
||||
<RETURN NAME="NB_FILE" TYPE="id_Integer" AVAILABILITY="AFTER"/>
|
||||
<RETURN NAME="CURRENT_STATUS" TYPE="id_String" AVAILABILITY="FLOW"/>
|
||||
|
||||
@@ -16,10 +16,23 @@ String dieOnError = ElementParameterParser.getValue(node, "__DIE_ON_ERROR__");
|
||||
String remotedir = ElementParameterParser.getValue(node, "__REMOTEDIR__");
|
||||
String connection = ElementParameterParser.getValue(node, "__CONNECTION__");
|
||||
String useExistingConn = ElementParameterParser.getValue(node, "__USE_EXISTING_CONNECTION__");
|
||||
boolean sftp = false;
|
||||
|
||||
boolean usePerl5 = ("true").equals(ElementParameterParser.getValue(node, "__PERL5_REGEX__"));
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
if ("true".equals(useExistingConn)) {
|
||||
List<? extends INode> nodeList = node.getProcess().getGeneratingNodes();
|
||||
|
||||
for (INode n : nodeList) {
|
||||
|
||||
if (n.getUniqueName().equals(connection)) {
|
||||
sftp = "true".equals(ElementParameterParser.getValue(n, "__SFTP__"));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
sftp = "true".equals(ElementParameterParser.getValue(node, "__SFTP__"));
|
||||
}
|
||||
%>
|
||||
|
||||
try {
|
||||
|
||||
@@ -62,10 +62,8 @@ if (("true").equals(useExistingConn)) {
|
||||
}
|
||||
} else {
|
||||
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
|
||||
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
|
||||
%>
|
||||
<%
|
||||
}
|
||||
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
|
||||
}
|
||||
|
||||
//The following part support the socks proxy for FTP and SFTP (Socks V4 or V5, they are all OK).
|
||||
//And it can not work with the FTP proxy directly, only support the socks proxy.
|
||||
@@ -152,8 +150,8 @@ int nb_file_<%=cid%> = 0;
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
|
||||
|
||||
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
|
||||
|
||||
<%if(use_encoding) {%>
|
||||
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
|
||||
@@ -230,103 +228,255 @@ int nb_file_<%=cid%> = 0;
|
||||
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
|
||||
|
||||
<%} else {
|
||||
if (!ftps) {%> // *** ftp *** //
|
||||
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
|
||||
<%
|
||||
if (("true").equals(useExistingConn)) {
|
||||
String conn= "conn_" + connection;
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%
|
||||
} else {
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
|
||||
<%
|
||||
}
|
||||
<%} else if (!ftps) {%> // *** ftp *** //
|
||||
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> =null;
|
||||
|
||||
if("binary".equalsIgnoreCase(ElementParameterParser.getValue(node, "__MODE__"))){
|
||||
%>
|
||||
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.ASCII_FILE_TYPE);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
// msg_<%=cid%> likes a String[] to save the message from transfer.
|
||||
java.util.List<String> msg_<%=cid%> = new java.util.ArrayList<>();
|
||||
<%
|
||||
} else { // *** ftps *** //
|
||||
String keystoreFile = ElementParameterParser.getValue(node,"__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
<%
|
||||
if(("false").equals(useExistingConn)){
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
|
||||
<%
|
||||
} else {
|
||||
String conn= "conn_" + connection;
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient)globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Uses an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
} //common code for ftp and ftps:
|
||||
%>
|
||||
String rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
|
||||
String remotedir_<%=cid%> = (<%=remotedir%>).replaceAll("\\\\", "/");
|
||||
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
|
||||
<%
|
||||
if(createDirIfNotExist) {
|
||||
%>
|
||||
if(!cwdSuccess_<%=cid %>){
|
||||
String[] dirsTree_<%=cid %> = remotedir_<%=cid %>.split("/");
|
||||
for (String dir: dirsTree_<%=cid %>) {
|
||||
ftp_<%=cid %>.makeDirectory(dir);
|
||||
ftp_<%=cid %>.changeWorkingDirectory(dir);
|
||||
<%
|
||||
if (("true").equals(useExistingConn)) {
|
||||
String conn= "conn_" + connection;
|
||||
%>
|
||||
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Uses an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
|
||||
cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
|
||||
<%}%>
|
||||
<%} else {%>
|
||||
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
|
||||
ftp_<%=cid %>.setRemoteHost(<%=host %>);
|
||||
ftp_<%=cid %>.setRemotePort(<%=port %>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
|
||||
<%}%>
|
||||
|
||||
<%if (("ACTIVE").equals(connectMode)) {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
|
||||
<%} else {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect();
|
||||
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
<%}%>
|
||||
|
||||
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
|
||||
// msg_<%=cid%> likes a String[] to save the message from transfer.
|
||||
com.enterprisedt.net.ftp.TransferCompleteStrings msg_<%=cid%> = ftp_<%=cid %>.getTransferCompleteMessages();
|
||||
msg_<%=cid%>.clearAll();
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
|
||||
|
||||
<%
|
||||
for (int i = 0; i < files.size(); i++) {
|
||||
Map<String, String> line = files.get(i);
|
||||
%>
|
||||
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
|
||||
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,<%= line.get("NEWNAME") %>);
|
||||
list<%=cid %>.add(map<%=cid %><%=i %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
String remotedir<%=cid %> = (<%=remotedir%>).replaceAll("\\\\", "/");
|
||||
<%
|
||||
if (createDirIfNotExist) {
|
||||
%>
|
||||
boolean isDirectoryExist_<%=cid %> = ftp_<%=cid %>.existsDirectory(remotedir<%=cid %>);
|
||||
|
||||
if (!isDirectoryExist_<%=cid %>) {
|
||||
String rootDir = null;
|
||||
String[] dirsTree_<%=cid %> = remotedir<%=cid %>.split("/");
|
||||
for (String dir: dirsTree_<%=cid %>) {
|
||||
if (!ftp_<%=cid %>.existsDirectory(dir)) {
|
||||
ftp_<%=cid %>.mkdir(dir);
|
||||
}
|
||||
ftp_<%=cid %>.chdir(dir);
|
||||
}
|
||||
|
||||
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
ftp_<%=cid %>.chdir(remotedir<%=cid %>);
|
||||
|
||||
<%if ("binary".equalsIgnoreCase(ElementParameterParser.getValue(node, "__MODE__"))) {%>
|
||||
ftp_<%=cid %>.setType(com.enterprisedt.net.ftp.FTPTransferType.BINARY);
|
||||
<%} else { %>
|
||||
ftp_<%=cid %>.setType(com.enterprisedt.net.ftp.FTPTransferType.ASCII);
|
||||
<%}%>
|
||||
String localdir<%=cid %> = <%=localdir%>;
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Uploading files to the server.");
|
||||
<%}%>
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
|
||||
<%
|
||||
} else { // *** ftps *** //
|
||||
String keystoreFile = ElementParameterParser.getValue(node,"__KEYSTORE_FILE__");
|
||||
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
String rootDir_<%=cid %> = null;
|
||||
<%
|
||||
if(("false").equals(useExistingConn)){
|
||||
%>
|
||||
<% /* create connection */ %>
|
||||
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
|
||||
|
||||
javax.net.ssl.SSLContext sslContext_<%=cid%> = null;
|
||||
javax.net.ssl.TrustManager[] trustManager_<%=cid%> = null;
|
||||
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid%> = null;
|
||||
|
||||
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
|
||||
|
||||
try {
|
||||
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
|
||||
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
|
||||
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
|
||||
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
|
||||
sslSocketFactory_<%=cid %> = sslContext_<%=cid%>.getSocketFactory();
|
||||
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
|
||||
if (isHTTPProxyUsed) {
|
||||
String httpProxyHost = System.getProperty("http.proxyHost");
|
||||
int httpProxyPort = Integer.getInteger("http.proxyPort");
|
||||
String httpProxyUser = System.getProperty("http.proxyUser");
|
||||
String httpProxyPass = System.getProperty("http.proxyPassword");
|
||||
|
||||
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
|
||||
} else {
|
||||
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
|
||||
}
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
if (!cwdSuccess_<%=cid %>) {
|
||||
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
|
||||
}
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
|
||||
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
|
||||
if (!isLoginSuccessful_<%=cid %>) {
|
||||
throw new RuntimeException("Login failed");
|
||||
}
|
||||
|
||||
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
|
||||
<%
|
||||
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
|
||||
<%
|
||||
}
|
||||
|
||||
if (protectionLevel != null && !protectionLevel.isEmpty()) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} catch (Exception e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
|
||||
<%}%>
|
||||
<%
|
||||
if (dieOnError) {
|
||||
%>
|
||||
throw e;
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
e.printStackTrace();
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
<%
|
||||
if ("PASSIVE".equals(connectMode)) {
|
||||
%>
|
||||
ftp_<%=cid %>.enterLocalPassiveMode();
|
||||
<%
|
||||
if (isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - Using the passive mode.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
String conn= "conn_" + connection;
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient)globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Uses an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
|
||||
rootDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
|
||||
<%
|
||||
}
|
||||
%>
|
||||
String remotedir_<%=cid%> = (<%=remotedir%>).replaceAll("\\\\", "/");
|
||||
|
||||
<%
|
||||
for (int i = 0; i < files.size(); i++) {
|
||||
Map<String, String> line = files.get(i);
|
||||
if(createDirIfNotExist) {
|
||||
%>
|
||||
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
|
||||
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>, <%= line.get("NEWNAME") %>);
|
||||
list<%=cid %>.add(map<%=cid %><%=i %>);
|
||||
String[] dirsTree_<%=cid %> = remotedir_<%=cid %>.split("/");
|
||||
for (String dir: dirsTree_<%=cid %>) {
|
||||
ftp_<%=cid %>.makeDirectory(dir);
|
||||
ftp_<%=cid %>.changeWorkingDirectory(dir);
|
||||
}
|
||||
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
String localdir<%=cid %> = <%=localdir%>;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Uploading files to the server.");
|
||||
<%}%>
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
|
||||
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
|
||||
|
||||
if (!cwdSuccess_<%=cid %>) {
|
||||
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
|
||||
}
|
||||
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
|
||||
|
||||
<%
|
||||
}
|
||||
for (int i = 0; i < files.size(); i++) {
|
||||
Map<String, String> line = files.get(i);
|
||||
%>
|
||||
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
|
||||
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>, <%= line.get("NEWNAME") %>);
|
||||
list<%=cid %>.add(map<%=cid %><%=i %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
String localdir<%=cid %> = <%=localdir%>;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Uploading files to the server.");
|
||||
<%}%>
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
|
||||
<%
|
||||
} /* close ftps */
|
||||
%>
|
||||
|
||||
@@ -57,9 +57,9 @@ if(sftp){ // *** sftp *** //
|
||||
<%}else if(!ftps){%> // *** ftp *** //
|
||||
}
|
||||
|
||||
msg_<%=cid%>.add(nb_file_<%=cid%> + " files have been uploaded.");
|
||||
msg_<%=cid%>.add(ftp_<%=cid %>.getUploadCount() + " files have been uploaded.");
|
||||
|
||||
String[] msgAll_<%=cid %> = msg_<%=cid%>.toArray(new String[0]);
|
||||
String[] msgAll_<%=cid %> = msg_<%=cid%>.getAll();
|
||||
StringBuffer sb_<%=cid %> = new StringBuffer();
|
||||
if (msgAll_<%=cid %> != null) {
|
||||
for (String item_<%=cid %> : msgAll_<%=cid %>) {
|
||||
@@ -76,13 +76,13 @@ if(sftp){ // *** sftp *** //
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
}catch(java.io.IOException ioe_<%=cid%>){
|
||||
}catch(java.net.SocketException se_<%=cid%>){
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
//ignore failure
|
||||
}
|
||||
@@ -92,7 +92,7 @@ if(sftp){ // *** sftp *** //
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
@@ -101,7 +101,7 @@ if(sftp){ // *** sftp *** //
|
||||
}else{
|
||||
if(!moveToCurrentDir){
|
||||
%>
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
|
||||
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
@@ -165,7 +165,7 @@ COMPATIBILITY="ALL"
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="6" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="6" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
|
||||
<DEFAULT>true</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
@@ -343,12 +343,12 @@ COMPATIBILITY="ALL"
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
|
||||
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
|
||||
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS>
|
||||
|
||||
@@ -185,14 +185,125 @@ try{
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
<%
|
||||
} else {// *** ftp+ ftps *** //
|
||||
if (!ftps) {
|
||||
ftpsoverwrite = overwrite; //to avoid migration, different dropdowns with same purpose
|
||||
}else if (!ftps) { // *** ftp *** //
|
||||
%>
|
||||
|
||||
String currentStatus_<%=cid %> = "No file transfered.";
|
||||
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
|
||||
for (String key<%=cid %> : keySet<%=cid %>){
|
||||
if(key<%=cid %> == null || "".equals(key<%=cid%>)){
|
||||
System.err.println("file name invalid!");
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - file name invalid!");
|
||||
<%}%>
|
||||
continue;
|
||||
}
|
||||
String tempdir<%=cid %> = localdir<%=cid %>;
|
||||
String filemask<%=cid %> = key<%=cid %>;
|
||||
String dir<%=cid %> = null;
|
||||
<%if(!useGlob) {//perl5 mode not support windows(\) path separator at the mask string%>
|
||||
String mask<%=cid %> = filemask<%=cid %>.replaceAll("\\\\", "/") ;
|
||||
<%} else {%>
|
||||
String mask<%=cid %> = filemask<%=cid%>;
|
||||
<%}%>
|
||||
int i<%=cid %> = mask<%=cid %>.lastIndexOf('/');
|
||||
if (i<%=cid %>!=-1){
|
||||
dir<%=cid %> = mask<%=cid %>.substring(0, i<%=cid %>);
|
||||
mask<%=cid %> = mask<%=cid %>.substring(i<%=cid %>+1);
|
||||
}
|
||||
if (dir<%=cid %>!=null && !"".equals(dir<%=cid %>)){
|
||||
tempdir<%=cid %> = tempdir<%=cid %> + "/" + dir<%=cid %>;
|
||||
}
|
||||
<%if(!useGlob) {%>
|
||||
mask<%=cid %> = mask<%=cid %>.replaceAll("\\.", "\\\\.").replaceAll("\\*", ".*");
|
||||
<%}%>
|
||||
final String finalMask<%=cid %> = mask<%=cid %>;
|
||||
java.io.File[] listings<%=cid %> = null;
|
||||
java.io.File file<%=cid %> = new java.io.File(tempdir<%=cid %>);
|
||||
if (file<%=cid %>.isDirectory()) {
|
||||
listings<%=cid %> = file<%=cid %>.listFiles(new java.io.FileFilter() {
|
||||
public boolean accept(java.io.File pathname) {
|
||||
boolean result = false;
|
||||
if (pathname != null && pathname.isFile()) {
|
||||
result = java.util.regex.Pattern.compile(finalMask<%=cid %>).matcher(pathname.getName()).find();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
});
|
||||
}
|
||||
if(listings<%=cid %> != null && listings<%=cid %>.length > 0){
|
||||
for (int m<%=cid %> = 0; m<%=cid %> < listings<%=cid %>.length; m<%=cid %>++){
|
||||
java.io.File file_in_localDir = listings<%=cid %>[m<%=cid %>];
|
||||
String uploadingFileName_<%= cid %> = file_in_localDir.getName();
|
||||
String newFileName_<%= cid %> = map<%=cid %>.get(key<%=cid %>);
|
||||
String newName_<%=cid%> = ("".equals(newFileName_<%= cid %>)) ? uploadingFileName_<%= cid %> : newFileName_<%= cid %>;
|
||||
boolean targetFileExistsOnRemoteServer_<%= cid %> = false;
|
||||
try{
|
||||
if (uploadingFileName_<%= cid %>.matches(mask<%=cid %>)){
|
||||
<%
|
||||
if(("never".equals(overwrite) || "size_differ".equals(overwrite)) && !append){
|
||||
%>
|
||||
targetFileExistsOnRemoteServer_<%= cid %> = ftp_<%= cid %>.existsFile(newName_<%= cid %>);
|
||||
globalMap.put("<%=cid %>_CURRENT_FILE_EXISTS", targetFileExistsOnRemoteServer_<%= cid %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%if(append){%>
|
||||
ftp_<%=cid %>.put(tempdir<%=cid %> + "/" + uploadingFileName_<%= cid %>, newName_<%= cid %>, true);
|
||||
<%}else if(("never").equals(overwrite)){%>
|
||||
if(!targetFileExistsOnRemoteServer_<%= cid %>){
|
||||
ftp_<%=cid %>.put(tempdir<%=cid %> + "/" + uploadingFileName_<%= cid %>, newName_<%=cid%>);
|
||||
}
|
||||
<%}else if(("always").equals(overwrite)){%>
|
||||
ftp_<%=cid %>.put(tempdir<%=cid %> + "/" + uploadingFileName_<%= cid %>, newName_<%=cid%>);
|
||||
<%}else if(("size_differ").equals(overwrite)){%>
|
||||
if (targetFileExistsOnRemoteServer_<%= cid %>){
|
||||
com.enterprisedt.net.ftp.FTPFile ftpfile<%=cid%>=ftp_<%=cid%>.fileDetails(newName_<%=cid%>);
|
||||
long ftpSize<%=cid%>=ftpfile<%=cid%>.size();
|
||||
long localSize<%=cid%>=file_in_localDir.length();
|
||||
if(ftpSize<%=cid%> != localSize<%=cid%>){
|
||||
ftp_<%=cid %>.put(tempdir<%=cid %> + "/" + uploadingFileName_<%= cid %>, newName_<%=cid%>);
|
||||
}
|
||||
}else{
|
||||
ftp_<%=cid %>.put(tempdir<%=cid %> + "/" + uploadingFileName_<%= cid %>, newName_<%=cid%>);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
currentStatus_<%=cid %> = "File rename OK.";
|
||||
msg_<%=cid%>.add("file: " + file_in_localDir.getAbsolutePath() + ", size: "
|
||||
+ file_in_localDir.length() + " bytes upload successfully");
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Uploaded file '" + uploadingFileName_<%= cid %> + "' successfully.");
|
||||
<%}%>
|
||||
|
||||
nb_file_<%=cid%>++;
|
||||
}
|
||||
}catch(com.enterprisedt.net.ftp.FTPException e_<%=cid%>){
|
||||
|
||||
msg_<%=cid%>.add("file " + file_in_localDir.getAbsolutePath() + " not found?");
|
||||
|
||||
currentStatus_<%=cid %> = "File rename fail.";
|
||||
globalMap.put("<%=cid %>_CURRENT_FILE_EXISTS", targetFileExistsOnRemoteServer_<%= cid %>);
|
||||
throw e_<%=cid%>;
|
||||
}
|
||||
}
|
||||
}else{
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - No match file exist!");
|
||||
<%}%>
|
||||
System.err.println("No match file exist!");
|
||||
}
|
||||
|
||||
}
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", currentStatus_<%=cid %>);
|
||||
|
||||
<%
|
||||
} else {// *** ftps *** //
|
||||
%>
|
||||
String currentStatus_<%=cid %> = "No file transfered.";
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", currentStatus_<%=cid %>);
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file transfered.");
|
||||
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
|
||||
boolean needRename_<%=cid%> = false;
|
||||
for (String key<%=cid %> : keySet<%=cid %>){
|
||||
@@ -257,43 +368,17 @@ try{
|
||||
java.io.FileInputStream file_stream_<%=cid %> = new java.io.FileInputStream(file_in_localDir_<%=cid%>);
|
||||
globalMap.put("<%=cid %>_CURRENT_FILE_EXISTS", remoteExistsFiles_<%=cid%>.contains(file_in_localDir_<%=cid%>.getName()));
|
||||
String newName_<%=cid%> = ("".equals(map<%=cid %>.get(key<%=cid %>)))?file_in_localDir_<%=cid%>.getName():map<%=cid %>.get(key<%=cid %>);
|
||||
needRename_<%=cid%> = true;
|
||||
<%
|
||||
if (!ftps && append) {
|
||||
%>
|
||||
if ((remoteExistsFiles_<%=cid%>.contains(newName_<%=cid%>))){
|
||||
ftp_<%=cid %>.appendFile(file_in_localDir_<%=cid%>.getName(), file_stream_<%=cid %>);
|
||||
} else {
|
||||
ftp_<%=cid %>.storeFile(file_in_localDir_<%=cid%>.getName(), file_stream_<%=cid %>);
|
||||
}
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Uploaded file '" + newName_<%=cid%> + "' successfully.");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_FILE_EXISTS", remoteExistsFiles_<%=cid%>.contains(newName_<%=cid%>));
|
||||
<%
|
||||
} else if ("size_differ".equals(ftpsoverwrite)) {
|
||||
%>
|
||||
if ((remoteExistsFiles_<%=cid%>.contains(newName_<%=cid%>))){
|
||||
long ftpSize = java.util.Arrays.stream(ftp_<%=cid %>.listFiles(newName_<%=cid%>)).filter(org.apache.commons.net.ftp.FTPFile::isFile).findFirst().get().getSize();
|
||||
long localSize = file_in_localDir_<%=cid%>.length();
|
||||
if (ftpSize != localSize) {
|
||||
ftp_<%=cid %>.deleteFile(newName_<%=cid%>);
|
||||
}
|
||||
}
|
||||
ftp_<%=cid %>.storeFile(file_in_localDir_<%=cid%>.getName(), file_stream_<%=cid %>);
|
||||
<%
|
||||
} else if("never".equals(ftpsoverwrite)){
|
||||
%>
|
||||
<%if("never".equals(ftpsoverwrite)){%>
|
||||
if (!(remoteExistsFiles_<%=cid%>.contains(newName_<%=cid%>))){
|
||||
needRename_<%=cid%> = true;
|
||||
ftp_<%=cid %>.storeFile(file_in_localDir_<%=cid%>.getName(), file_stream_<%=cid %>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Uploaded file '" + newName_<%=cid%> + "' successfully.");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_FILE_EXISTS", remoteExistsFiles_<%=cid%>.contains(newName_<%=cid%>));
|
||||
} else {
|
||||
needRename_<%=cid %> = false;
|
||||
}
|
||||
<%}else if("always".equals(ftpsoverwrite)){%>
|
||||
needRename_<%=cid%> = true;
|
||||
if ((remoteExistsFiles_<%=cid%>.contains(newName_<%=cid%>))){
|
||||
ftp_<%=cid %>.deleteFile(newName_<%=cid%>);
|
||||
}
|
||||
@@ -306,14 +391,6 @@ try{
|
||||
|
||||
<%}%>
|
||||
file_stream_<%=cid %>.close();
|
||||
<%
|
||||
if (!ftps) {
|
||||
%>
|
||||
msg_<%=cid%>.add("file: " + file_in_localDir_<%=cid%>.getAbsolutePath() + ", size: "
|
||||
+ file_in_localDir_<%=cid%>.length() + " bytes upload successfully");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
nb_file_<%=cid%>++;
|
||||
}
|
||||
}
|
||||
@@ -341,12 +418,6 @@ try{
|
||||
<%}%>
|
||||
}catch(java.lang.Exception e_<%=cid%>){
|
||||
<%
|
||||
if (!ftps) {
|
||||
%>
|
||||
msg_<%=cid%>.add("file not found?: " + e_<%=cid%>.getMessage());
|
||||
<%
|
||||
|
||||
}
|
||||
if(("true").equals(dieOnError)){
|
||||
%>
|
||||
throw(e_<%=cid%>);
|
||||
|
||||
@@ -57,8 +57,6 @@ if (("true").equals(useExistingConn)) {
|
||||
} else {
|
||||
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
|
||||
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
|
||||
%>
|
||||
<%
|
||||
}
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
@@ -133,9 +131,9 @@ if (sftp) { // *** sftp *** //
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
|
||||
|
||||
|
||||
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
|
||||
|
||||
<%if(use_encoding) {%>
|
||||
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
|
||||
<%}%>
|
||||
@@ -174,42 +172,102 @@ if (sftp) { // *** sftp *** //
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
|
||||
|
||||
<%
|
||||
} else {
|
||||
if (ftps) { // *** ftps *** //
|
||||
} else if (ftps) { // *** ftps *** //
|
||||
if (("false").equals(useExistingConn)) {
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
|
||||
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
|
||||
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
|
||||
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
|
||||
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
|
||||
try {
|
||||
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
|
||||
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
|
||||
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
|
||||
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
|
||||
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
|
||||
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
|
||||
if (isHTTPProxyUsed) {
|
||||
String httpProxyHost = System.getProperty("http.proxyHost");
|
||||
int httpProxyPort = Integer.getInteger("http.proxyPort");
|
||||
String httpProxyUser = System.getProperty("http.proxyUser");
|
||||
String httpProxyPass = System.getProperty("http.proxyPassword");
|
||||
|
||||
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
|
||||
} else {
|
||||
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
|
||||
}
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
|
||||
<%
|
||||
if (("false").equals(useExistingConn)) {
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
|
||||
if (!isLoginSuccessful_<%=cid %>) {
|
||||
throw new RuntimeException("Login failed");
|
||||
}
|
||||
|
||||
<%
|
||||
} else {
|
||||
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
|
||||
<%
|
||||
}
|
||||
} else { // *** ftp *** //
|
||||
}
|
||||
|
||||
if (protectionLevel != null && !protectionLevel.isEmpty()) {
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
|
||||
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
|
||||
<%
|
||||
if (("true").equals(useExistingConn)) {
|
||||
}
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
} catch (Exception e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
|
||||
<%}%>
|
||||
<%
|
||||
} else {
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
if (dieOnError) {
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
|
||||
throw e;
|
||||
<%
|
||||
}
|
||||
} //common code for ftp and ftps:
|
||||
} else {
|
||||
%>
|
||||
e.printStackTrace();
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
<%
|
||||
if ("PASSIVE".equals(connectMode)) {
|
||||
%>
|
||||
ftp_<%=cid %>.enterLocalPassiveMode();
|
||||
<%
|
||||
if (isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - Using the passive mode.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
|
||||
|
||||
@@ -236,5 +294,67 @@ if (sftp) { // *** sftp *** //
|
||||
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) { <% /* start main loop */ %>
|
||||
<%
|
||||
}
|
||||
} else {// *** ftp *** //
|
||||
%>
|
||||
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> =null;
|
||||
|
||||
<%if (("true").equals(useExistingConn)) {%>
|
||||
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%if(!moveToCurrentDir){%>
|
||||
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
|
||||
<%}%>
|
||||
<%} else {%>
|
||||
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
|
||||
ftp_<%=cid %>.setRemoteHost(<%=host %>);
|
||||
ftp_<%=cid %>.setRemotePort(<%=port %>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
|
||||
<%}%>
|
||||
<%if (("ACTIVE").equals(connectMode)) {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
|
||||
<%} else {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect();
|
||||
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
<%}%>
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
|
||||
|
||||
<%
|
||||
for (int i = 0; i < files.size(); i++) {
|
||||
Map<String, String> line = files.get(i);
|
||||
%>
|
||||
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
|
||||
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,<%= line.get("NEWNAME") %>);
|
||||
list<%=cid %>.add(map<%=cid %><%=i %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
String remotedir<%=cid %> = <%=remotedir%>;
|
||||
ftp_<%=cid %>.chdir(remotedir<%=cid %>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Renaming file from server.");
|
||||
<%}%>
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
|
||||
<%}%>
|
||||
|
||||
@@ -78,13 +78,13 @@
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
}catch(java.io.IOException ioe_<%=cid%>){
|
||||
}catch(java.net.SocketException se_<%=cid%>){
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
//ignore failure
|
||||
}
|
||||
@@ -94,7 +94,7 @@
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
@@ -103,7 +103,7 @@
|
||||
}else{
|
||||
if(!moveToCurrentDir){
|
||||
%>
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
|
||||
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,7 +143,7 @@ COMPATIBILITY="ALL"
|
||||
<DEFAULT>"."</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="31" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="31" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
|
||||
<DEFAULT>true</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
@@ -261,12 +261,12 @@ COMPATIBILITY="ALL"
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
|
||||
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS>
|
||||
|
||||
@@ -57,74 +57,25 @@ try{
|
||||
}
|
||||
|
||||
<%
|
||||
} else { // *** ftp+s *** //
|
||||
} else if (ftps) { // *** ftps *** //
|
||||
%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file renamed.");
|
||||
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
|
||||
for (String key<%=cid %> : keySet<%=cid %>) {
|
||||
try{
|
||||
<%
|
||||
if (!ftps && "never".equals(overwrite)) {
|
||||
%>
|
||||
String[] remoteFileArray_<%=cid%> = java.util.Arrays.stream(ftp_<%=cid %>.listFiles()).filter(org.apache.commons.net.ftp.FTPFile::isFile).map(org.apache.commons.net.ftp.FTPFile::getName).filter(map<%=cid %>.get(key<%=cid %>)::equals).toArray(String[]::new);
|
||||
boolean isRemoteFileExists_<%=cid %> = (remoteFileArray_<%=cid%> != null) && (remoteFileArray_<%=cid%>.length > 0);
|
||||
if (!isRemoteFileExists_<%=cid%>) {
|
||||
boolean renameSuccess_<%=cid %> = ftp_<%=cid%>.rename(key<%=cid %>, map<%=cid %>.get(key<%=cid %>));
|
||||
if (renameSuccess_<%=cid %>) {
|
||||
nb_file_<%=cid%>++;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - '" + key<%=cid %> + "' rename ok.");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
|
||||
} else {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - '" + key<%=cid %> + "' rename failed. " + ftp_<%=cid %>.getReplyString());
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename fail.");
|
||||
}
|
||||
}
|
||||
<%
|
||||
} else if (!ftps && "size_differ".equals(overwrite)) {
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPFile toRenameFile_<%=cid %> = java.util.Arrays.stream(ftp_<%=cid %>.listFiles(key<%=cid %>)).filter(org.apache.commons.net.ftp.FTPFile::isFile).findFirst().orElse(null);
|
||||
if (toRenameFile_<%=cid %> != null) {
|
||||
org.apache.commons.net.ftp.FTPFile toOverwriteFile_<%=cid %> = java.util.Arrays.stream(ftp_<%=cid %>.listFiles(map<%=cid %>.get(key<%=cid %>))).filter(org.apache.commons.net.ftp.FTPFile::isFile).findFirst().orElse(null);
|
||||
if ((toOverwriteFile_<%=cid %> != null) && toRenameFile_<%=cid %>.getSize() != toOverwriteFile_<%=cid%>.getSize()) {
|
||||
ftp_<%=cid %>.deleteFile(map<%=cid %>.get(key<%=cid %>));
|
||||
}
|
||||
boolean renameSuccess_<%=cid %> = ftp_<%=cid%>.rename(key<%=cid %>, map<%=cid %>.get(key<%=cid %>));
|
||||
if (renameSuccess_<%=cid %>) {
|
||||
nb_file_<%=cid%>++;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - '" + key<%=cid %> + "' rename ok.");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
|
||||
} else {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - '" + key<%=cid %> + "' rename failed. " + ftp_<%=cid %>.getReplyString());
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename fail.");
|
||||
}
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
boolean renameSuccess_<%=cid %> = ftp_<%=cid%>.rename(key<%=cid %>, map<%=cid %>.get(key<%=cid %>));
|
||||
if (renameSuccess_<%=cid %>) {
|
||||
nb_file_<%=cid%>++;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - '" + key<%=cid %> + "' rename ok.");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
|
||||
} else {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - '" + key<%=cid %> + "' rename failed. " + ftp_<%=cid %>.getReplyString());
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename fail.");
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
boolean renameSuccess_<%=cid %> = ftp_<%=cid%>.rename(key<%=cid %>, map<%=cid %>.get(key<%=cid %>));
|
||||
if (renameSuccess_<%=cid %>) {
|
||||
nb_file_<%=cid%>++;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - '" + key<%=cid %> + "' rename ok.");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
|
||||
} else {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - '" + key<%=cid %> + "' rename failed. " + ftp_<%=cid %>.getReplyString());
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename fail.");
|
||||
}
|
||||
}catch(IOException e){
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename fail.");
|
||||
<%
|
||||
@@ -143,21 +94,100 @@ try{
|
||||
}
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} catch (Exception e) {
|
||||
} else {%>//normal ftp
|
||||
|
||||
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
|
||||
String tmp_<%=cid%> = pid + System.currentTimeMillis();
|
||||
try{
|
||||
for (String key<%=cid %> : keySet<%=cid %>) {
|
||||
<%if(("always").equals(overwrite)){%>
|
||||
|
||||
String fromFile_<%=cid%> = key<%=cid %>;
|
||||
String toFile_<%=cid%> = map<%=cid %>.get(key<%=cid %>);
|
||||
|
||||
if (!fromFile_<%=cid%>.equals(toFile_<%=cid%>)){
|
||||
if(ftp_<%=cid%>.exists(fromFile_<%=cid%>)){
|
||||
if(ftp_<%=cid%>.exists(toFile_<%=cid%> + "." + tmp_<%=cid%>)){
|
||||
ftp_<%=cid%>.delete(toFile_<%=cid%> + "." + tmp_<%=cid%>);
|
||||
}
|
||||
if(ftp_<%=cid%>.exists(toFile_<%=cid%>)){
|
||||
ftp_<%=cid %>.rename(toFile_<%=cid%>, toFile_<%=cid%> + "." + tmp_<%=cid%>);
|
||||
}
|
||||
ftp_<%=cid %>.rename(fromFile_<%=cid%>, toFile_<%=cid%>);
|
||||
if(ftp_<%=cid%>.exists(toFile_<%=cid%> + "." + tmp_<%=cid%>)){
|
||||
ftp_<%=cid%>.delete(toFile_<%=cid%> + "." + tmp_<%=cid%>);
|
||||
}
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - '" + toFile_<%=cid%> + "' rename ok");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
|
||||
nb_file_<%=cid%>++;
|
||||
}
|
||||
}
|
||||
<%}%>
|
||||
<%if(("never").equals(overwrite)){%>
|
||||
if(!(ftp_<%=cid%>.exists(map<%=cid %>.get(key<%=cid %>)))) {
|
||||
ftp_<%=cid %>.rename(key<%=cid %>, map<%=cid %>.get(key<%=cid %>));
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - '" + key<%=cid %> + "' rename ok");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
|
||||
nb_file_<%=cid%>++;
|
||||
}
|
||||
<%}%>
|
||||
<%if(("size_differ").equals(overwrite)){%>
|
||||
if((ftp_<%=cid%>.exists(map<%=cid %>.get(key<%=cid %>)))){
|
||||
com.enterprisedt.net.ftp.FTPFile ftpnewNamefile<%=cid%>=ftp_<%=cid%>.fileDetails(map<%=cid %>.get(key<%=cid %>));
|
||||
com.enterprisedt.net.ftp.FTPFile ftporigNamefile<%=cid%>=ftp_<%=cid%>.fileDetails(key<%=cid %>);
|
||||
long newNameSize<%=cid%>=ftpnewNamefile<%=cid%>.size();
|
||||
long origNameSize<%=cid%>=ftporigNamefile<%=cid%>.size();
|
||||
if(!(newNameSize<%=cid%>==origNameSize<%=cid%>)) {
|
||||
String fromFile_<%=cid%> = key<%=cid %>;
|
||||
String toFile_<%=cid%> = map<%=cid %>.get(key<%=cid %>);
|
||||
if(ftp_<%=cid%>.exists(fromFile_<%=cid%>)){
|
||||
if(ftp_<%=cid%>.exists(toFile_<%=cid%> + "." + tmp_<%=cid%>)){
|
||||
ftp_<%=cid%>.delete(toFile_<%=cid%> + "." + tmp_<%=cid%>);
|
||||
}
|
||||
if(ftp_<%=cid%>.exists(toFile_<%=cid%>)){
|
||||
ftp_<%=cid %>.rename(toFile_<%=cid%>, toFile_<%=cid%> + "." + tmp_<%=cid%>);
|
||||
}
|
||||
ftp_<%=cid %>.rename(fromFile_<%=cid%>, toFile_<%=cid%>);
|
||||
if(ftp_<%=cid%>.exists(toFile_<%=cid%> + "." + tmp_<%=cid%>)){
|
||||
ftp_<%=cid%>.delete(toFile_<%=cid%> + "." + tmp_<%=cid%>);
|
||||
}
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - '" + toFile_<%=cid%> + "' rename ok");
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
|
||||
nb_file_<%=cid%>++;
|
||||
}
|
||||
}
|
||||
}else{
|
||||
ftp_<%=cid %>.rename(key<%=cid %>, map<%=cid %>.get(key<%=cid %>));
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename OK.");
|
||||
nb_file_<%=cid%>++;
|
||||
}
|
||||
<%}%>
|
||||
}
|
||||
}catch(com.enterprisedt.net.ftp.FTPException e){
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File rename fail.");
|
||||
throw e;
|
||||
}
|
||||
|
||||
<%}%>
|
||||
}catch(java.lang.Exception e){
|
||||
<%
|
||||
if(("true").equals(dieOnError)){
|
||||
if(("true").equals(dieOnError)){
|
||||
%>
|
||||
throw(e);
|
||||
<%
|
||||
}else{
|
||||
}else{
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - " + e.getMessage());
|
||||
<%}%>
|
||||
System.err.print(e.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - " + e.getMessage());
|
||||
<%}%>
|
||||
System.err.print(e.getMessage());
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
}
|
||||
@@ -55,8 +55,6 @@ if (("true").equals(useExistingConn)) {
|
||||
} else {
|
||||
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
|
||||
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
|
||||
%>
|
||||
<%
|
||||
}
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
@@ -133,7 +131,7 @@ if (sftp) {// *** sftp *** //
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
|
||||
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
|
||||
|
||||
<%if(use_encoding) {%>
|
||||
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
|
||||
@@ -169,76 +167,179 @@ if (sftp) {// *** sftp *** //
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
|
||||
|
||||
<%
|
||||
} else {
|
||||
if (ftps) { // *** ftps *** //
|
||||
} else if (ftps) { // *** ftps *** //
|
||||
/* creating connection */
|
||||
if (("false").equals(useExistingConn)) {
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
<%
|
||||
/* creating connection */
|
||||
if (("false").equals(useExistingConn)) {
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
<%
|
||||
if (moveToCurrentDir) {
|
||||
%>
|
||||
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(<%=remotedir %>);
|
||||
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
|
||||
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
|
||||
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
|
||||
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
|
||||
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
|
||||
|
||||
if (!cwdSuccess_<%=cid %>) {
|
||||
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else { // *** ftp *** //
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPClient ftp_<%=cid %> = null;
|
||||
<%
|
||||
if (("true").equals(useExistingConn)) {
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPClient) globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteAddress().toString() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%
|
||||
if (moveToCurrentDir) {
|
||||
%>
|
||||
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(<%=remotedir %>);
|
||||
try {
|
||||
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
|
||||
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
|
||||
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
|
||||
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
|
||||
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
|
||||
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
|
||||
if (isHTTPProxyUsed) {
|
||||
String httpProxyHost = System.getProperty("http.proxyHost");
|
||||
int httpProxyPort = Integer.getInteger("http.proxyPort");
|
||||
String httpProxyUser = System.getProperty("http.proxyUser");
|
||||
String httpProxyPass = System.getProperty("http.proxyPassword");
|
||||
|
||||
if (!cwdSuccess_<%=cid %>) {
|
||||
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
|
||||
}
|
||||
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
|
||||
} else {
|
||||
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
|
||||
}
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
|
||||
<%
|
||||
}
|
||||
} else {
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%
|
||||
}
|
||||
} //common code for ftp and ftps:
|
||||
}
|
||||
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
<% /*store real pwd*/ %>
|
||||
String rootWorkDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
|
||||
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
|
||||
if (!isLoginSuccessful_<%=cid %>) {
|
||||
throw new RuntimeException("Login failed");
|
||||
}
|
||||
<%
|
||||
for (int i = 0; i < files.size(); i++) {
|
||||
Map<String, String> line = files.get(i);
|
||||
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
|
||||
%>
|
||||
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
|
||||
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,"");
|
||||
list<%=cid %>.add(map<%=cid %><%=i %>);
|
||||
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
if (protectionLevel != null && !protectionLevel.isEmpty()) {
|
||||
%>
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) { <% /* start main loop */ %>
|
||||
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
} catch (Exception e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
|
||||
<%}%>
|
||||
throw e;
|
||||
}
|
||||
<%
|
||||
if ("PASSIVE".equals(connectMode)) {
|
||||
%>
|
||||
ftp_<%=cid %>.enterLocalPassiveMode();
|
||||
<%
|
||||
if (isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - Using the passive mode.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
<%
|
||||
if (moveToCurrentDir) {
|
||||
%>
|
||||
boolean cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(<%=remotedir %>);
|
||||
|
||||
if (!cwdSuccess_<%=cid %>) {
|
||||
throw new RuntimeException("Failed to change remote directory. " + ftp_<%=cid %>.getReplyString());
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
<% /*store real pwd*/ %>
|
||||
String rootWorkDir_<%=cid %> = ftp_<%=cid %>.printWorkingDirectory();
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
|
||||
<%
|
||||
for (int i = 0; i < files.size(); i++) {
|
||||
Map<String, String> line = files.get(i);
|
||||
%>
|
||||
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
|
||||
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,"");
|
||||
list<%=cid %>.add(map<%=cid %><%=i %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) { <% /* start main loop */ %>
|
||||
<%
|
||||
} else {// *** ftp *** //
|
||||
%>
|
||||
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> = null;
|
||||
|
||||
<%if (("true").equals(useExistingConn)) {%>
|
||||
ftp_<%=cid %> = (com.enterprisedt.net.ftp.FTPClient)globalMap.get("<%=conn %>");
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(ftp_<%=cid %>!=null) {
|
||||
log.info("<%=cid%> - Use an existing connection. Connection hostname: " + ftp_<%=cid %>.getRemoteHost() + ", Connection port: " + ftp_<%=cid %>.getRemotePort() + ".");
|
||||
}
|
||||
<%}%>
|
||||
<%if(!moveToCurrentDir){%>
|
||||
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
|
||||
<%}%>
|
||||
<%} else {%>
|
||||
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
|
||||
ftp_<%=cid %>.setRemoteHost(<%=host %>);
|
||||
ftp_<%=cid %>.setRemotePort(<%=port %>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
|
||||
<%}%>
|
||||
<% if (("ACTIVE").equals(connectMode)) {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
|
||||
<%} else {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect();
|
||||
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
<%}%>
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
|
||||
|
||||
<%for (int i = 0; i < files.size(); i++) {
|
||||
Map<String, String> line = files.get(i);
|
||||
%>
|
||||
java.util.Map<String,String> map<%=cid %><%=i %> = new java.util.HashMap<String,String>();
|
||||
map<%=cid %><%=i %>.put(<%= line.get("FILEMASK") %>,"");
|
||||
list<%=cid %>.add(map<%=cid %><%=i %>);
|
||||
<%}%>
|
||||
String remotedir<%=cid %> = <%=remotedir%>;
|
||||
ftp_<%=cid %>.chdir(remotedir<%=cid %>);
|
||||
String root<%=cid %> = ftp_<%=cid %>.pwd();
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Truncating file from the server.");
|
||||
<%}%>
|
||||
for (java.util.Map<String, String> map<%=cid %> : list<%=cid %>) {
|
||||
<%}%>
|
||||
|
||||
@@ -67,7 +67,7 @@
|
||||
}
|
||||
} else {// *** ftp *** //
|
||||
%>
|
||||
} <% /* close main loop */ %>
|
||||
}
|
||||
<%
|
||||
if(!("true").equals(useExistingConn)){
|
||||
if(("true").equals(ignoreFailureAtQuit)){
|
||||
@@ -76,13 +76,13 @@
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
}catch(java.io.IOException ioe_<%=cid%>){
|
||||
}catch(java.net.SocketException se_<%=cid%>){
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
log.warn("<%=cid%> - " + se_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
//ignore failure
|
||||
}
|
||||
@@ -92,12 +92,18 @@
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.quit();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
}else{
|
||||
if(!moveToCurrentDir){
|
||||
%>
|
||||
ftp_<%=cid %>.chdir(rootDir_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -149,7 +149,7 @@ COMPATIBILITY="ALL"
|
||||
<DEFAULT>"."</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="31" SHOW_IF="(USE_EXISTING_CONNECTION == 'true') AND (#LINK@NODE.CONNECTION.SFTP == 'false')">
|
||||
<PARAMETER NAME="MOVE_TO_THE_CURRENT_DIRECTORY" FIELD="CHECK" NUM_ROW="31" SHOW_IF="USE_EXISTING_CONNECTION == 'true'">
|
||||
<DEFAULT>true</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
@@ -249,13 +249,13 @@ COMPATIBILITY="ALL"
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true') AND (USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="jakarta-oro" MODULE="jakarta-oro-2.0.8.jar" MVN="mvn:org.talend.libraries/jakarta-oro-2.0.8/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache/lib/jakarta-oro-2.0.8.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="Java-FTP+S" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="FTP+S-Proxy-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar"
|
||||
MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819"
|
||||
REQUIRED_IF="(SFTP == 'false') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="talend-proxy" MODULE="talend-proxy-1.0.1.jar" MVN="mvn:org.talend.components.lib/talend-proxy/1.0.1" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
REQUIRED_IF="(FTPS == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS>
|
||||
|
||||
@@ -73,10 +73,10 @@
|
||||
}
|
||||
}
|
||||
<%
|
||||
} else { /* ftp+s */
|
||||
} else if (ftps) { /* ftps */
|
||||
%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file truncated.");
|
||||
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
|
||||
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
|
||||
for (String key<%=cid %> : keySet<%=cid %>) {
|
||||
<% /*prepare filemask*/ %>
|
||||
String filemask<%=cid %> = key<%=cid %>;
|
||||
@@ -188,4 +188,44 @@
|
||||
|
||||
<% /*return to root dir for next filemask*/ %>
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootWorkDir_<%=cid %>);
|
||||
<%} %>
|
||||
<%
|
||||
} else { /* ftp */
|
||||
%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file truncated.");
|
||||
java.util.Set<String> keySet<%=cid %> = map<%=cid %>.keySet();
|
||||
try {
|
||||
for (String key<%=cid %> : keySet<%=cid %>) {
|
||||
String filemask<%=cid %> = key<%=cid %>;
|
||||
String dir<%=cid %> = null;
|
||||
<%if(!useGlob) {//perl5 mode not support windows(\) path separator at the mask string%>
|
||||
String mask<%=cid %> = filemask<%=cid %>.replaceAll("\\\\", "/") ;
|
||||
<%}else{%>
|
||||
String mask<%=cid %> = filemask<%=cid %>;
|
||||
<%}%>
|
||||
int i<%=cid %> = mask<%=cid %>.lastIndexOf('/');
|
||||
if (i<%=cid %>!=-1) {
|
||||
dir<%=cid %> = mask<%=cid %>.substring(0, i<%=cid %>);
|
||||
mask<%=cid %> = mask<%=cid %>.substring(i<%=cid %>+1);
|
||||
}
|
||||
if (dir<%=cid %>!=null && !"".equals(dir<%=cid %>)) ftp_<%=cid %>.chdir(dir<%=cid %>);
|
||||
<%if(!useGlob) {%>
|
||||
mask<%=cid %> = org.apache.oro.text.GlobCompiler.globToPerl5(mask<%=cid %>.toCharArray(), org.apache.oro.text.GlobCompiler.DEFAULT_MASK);
|
||||
<%}%>
|
||||
String[] listings<%=cid %> = ftp_<%=cid %>.dir(null, false);
|
||||
for (int m<%=cid %> = 0; m<%=cid %> < listings<%=cid %>.length; m<%=cid %>++) {
|
||||
if (listings<%=cid %>[m<%=cid %>].matches(mask<%=cid %>)) {
|
||||
ftp_<%=cid %>.put(new ByteArrayInputStream(new byte[]{}), listings<%=cid %>[m<%=cid %>]);
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File truncate OK.");
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - '" + listings<%=cid %>[m<%=cid %>] + "' truncate ok.");
|
||||
<%}%>
|
||||
nb_file_<%=cid%>++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(com.enterprisedt.net.ftp.FTPException e){
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File truncate fail.");
|
||||
throw e;
|
||||
}
|
||||
ftp_<%=cid %>.chdir(root<%=cid %>);
|
||||
<%}%>
|
||||
|
||||
@@ -131,7 +131,7 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="commons-compress" MODULE="commons-compress-1.19.jar" MVN="mvn:org.apache.commons/commons-compress/1.19" REQUIRED="true" />
|
||||
<IMPORT NAME="COMMONS-COMPRESS-1.10" MODULE="commons-compress-1.10.jar" MVN="mvn:org.talend.libraries/commons-compress-1.10/6.1.0" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-compress-1.10.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="Encrypt-Zip" MODULE="checkArchive-1.1-20181130.jar" MVN="mvn:org.talend.libraries/checkArchive-1.1-20181130/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/checkArchive-1.1-20181130.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="zip4j_1.3.1.jar" MODULE="zip4j_1.3.1.jar" MVN="mvn:org.talend.libraries/zip4j_1.3.1/6.0.0" REQUIRED="true" />
|
||||
<IMPORT NAME="talendzip-1.0-20190527.jar" MODULE="talendzip-1.0-20190527.jar" MVN="mvn:org.talend.libraries/talendzip/1.0-20190527" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talendzip-1.0-20190527.jar" REQUIRED="true" />
|
||||
|
||||
@@ -464,7 +464,6 @@
|
||||
log.error("<%=cid%> - " +e.getMessage());
|
||||
<%}%>
|
||||
System.err.println(e.getMessage());
|
||||
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
|
||||
<% } %>
|
||||
}
|
||||
java.util.zip.ZipEntry entry_<%=cid %> = null;
|
||||
@@ -480,7 +479,6 @@
|
||||
log.error("<%=cid%> - " +e.getMessage());
|
||||
<%}%>
|
||||
System.err.println(e.getMessage());
|
||||
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
|
||||
break;
|
||||
<% } %>
|
||||
}
|
||||
@@ -505,7 +503,6 @@
|
||||
log.error("<%=cid%> - " +e.getMessage());
|
||||
<%}%>
|
||||
System.err.println(e.getMessage());
|
||||
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
|
||||
<% } %>
|
||||
}
|
||||
<%
|
||||
@@ -531,7 +528,6 @@
|
||||
log.error("<%=cid%> - " +e.getMessage());
|
||||
<%}%>
|
||||
System.err.println(e.getMessage());
|
||||
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
|
||||
<% } %>
|
||||
}
|
||||
<%
|
||||
@@ -610,9 +606,7 @@
|
||||
} else if(rejectConnName.equals(firstConnName)){%>
|
||||
<%=rejectConnName%>.errorMessage = e.getMessage() + " - Line: " + tos_count_<%=node.getUniqueName() %>;
|
||||
<%
|
||||
} %>
|
||||
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
}
|
||||
@@ -1082,7 +1076,6 @@
|
||||
log.error("<%=cid%> - " + e.getMessage());
|
||||
<%}%>
|
||||
System.err.println(e.getMessage());
|
||||
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
|
||||
<% } %>
|
||||
}
|
||||
java.util.zip.ZipEntry entry_<%=cid %> = null;
|
||||
@@ -1098,7 +1091,6 @@
|
||||
log.error("<%=cid%> - " + e.getMessage());
|
||||
<%}%>
|
||||
System.err.println(e.getMessage());
|
||||
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
|
||||
break;
|
||||
<% } %>
|
||||
}
|
||||
@@ -1293,7 +1285,6 @@
|
||||
log.error("<%=cid%> - " +e.getMessage());
|
||||
<%}%>
|
||||
System.err.println(e.getMessage());
|
||||
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
|
||||
<% } %>
|
||||
}//TD110 end
|
||||
|
||||
@@ -1397,9 +1388,7 @@
|
||||
%>
|
||||
<%=rejectConnName%>.errorMessage = e.getMessage() + " - Line: " + tos_count_<%=node.getUniqueName() %>;
|
||||
<%
|
||||
} %>
|
||||
globalMap.put("<%=cid %>_ERROR_MESSAGE", e.getMessage());
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
}
|
||||
|
||||
@@ -213,7 +213,7 @@
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Talen File Enhanced"
|
||||
MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar" REQUIRED="true" />
|
||||
MODULE="talend_file_enhanced_20070724.jar" MVN="mvn:org.talend.libraries/talend_file_enhanced_20070724/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced_20070724.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="Talend_CSV" MODULE="talendcsv.jar" MVN="mvn:org.talend.libraries/talendcsv/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.csv/lib/talendcsv.jar"
|
||||
REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
|
||||
@@ -185,7 +185,7 @@
|
||||
<IMPORT NAME="simpleexcel" MODULE="simpleexcel-2.1-20190507.jar" MVN="mvn:org.talend.libraries/simpleexcel-2.1-20190507/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.excel/lib/simpleexcel-2.1-20190507.jar" REQUIRED_IF="(VERSION_2007 == 'true') AND GENERATION_MODE == 'EVENT_MODE')" />
|
||||
<IMPORT NAME="commons-collections4" MODULE="commons-collections4-4.1.jar" MVN="mvn:org.talend.libraries/commons-collections4-4.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.excel/lib/commons-collections4-4.1.jar" REQUIRED_IF="(VERSION_2007 == 'true')" />
|
||||
|
||||
<IMPORT NAME="commons-compress" MODULE="commons-compress-1.19.jar" MVN="mvn:org.apache.commons/commons-compress/1.19" REQUIRED_IF="(VERSION_2007 == 'true')" />
|
||||
<IMPORT NAME="commons-compress" MODULE="commons-compress-1.18.jar" MVN="mvn:org.apache.commons/commons-compress/1.18" REQUIRED_IF="(VERSION_2007 == 'true')" />
|
||||
<IMPORT NAME="commons-math3" MODULE="commons-math3-3.6.1.jar" MVN="mvn:org.apache.commons/commons-math3/3.6.1" REQUIRED_IF="(VERSION_2007 == 'true')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -131,7 +131,7 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Talend File Enhanced" MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="Talend File Enhanced" MODULE="talend_file_enhanced_20070724.jar" MVN="mvn:org.talend.libraries/talend_file_enhanced_20070724/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced_20070724.jar" REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -165,7 +165,7 @@
|
||||
REQUIRED_IF="(READ_BY == 'XPATH')" BundleID="" />
|
||||
<IMPORT NAME="Java_JAXEN1.1" MODULE="jaxen-1.1.1.jar" MVN="mvn:org.talend.libraries/jaxen-1.1.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.dom4j-jaxen/lib/jaxen-1.1.1.jar"
|
||||
REQUIRED_IF="(READ_BY == 'XPATH')" BundleID="" />
|
||||
<IMPORT NAME="json-lib" MODULE="json-lib-2.4.5-talend.jar" MVN="mvn:net.sf.json-lib/json-lib/2.4.5-talend" REQUIRED_IF="(READ_BY == 'XPATH')" />
|
||||
<IMPORT NAME="json-lib" MODULE="json-lib-2.4.2-talend.jar" MVN="mvn:net.sf.json-lib/json-lib/2.4.2-talend" REQUIRED_IF="(READ_BY == 'XPATH')" />
|
||||
<IMPORT NAME="commons_lang" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar"
|
||||
REQUIRED_IF="(READ_BY == 'XPATH')" />
|
||||
<IMPORT NAME="commons_logging" MODULE="commons-logging-1.1.1.jar" MVN="mvn:org.talend.libraries/commons-logging-1.1.1/6.0.0" UrlPath="platform:/base/plugins/org.apache.commons.logging_1.1.1.v201101211721.jar"
|
||||
|
||||
@@ -118,7 +118,7 @@
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Talen File Enhanced"
|
||||
MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar" REQUIRED="true" />
|
||||
MODULE="talend_file_enhanced_20070724.jar" MVN="mvn:org.talend.libraries/talend_file_enhanced_20070724/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced_20070724.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="Talend_CSV" MODULE="talendcsv.jar" MVN="mvn:org.talend.libraries/talendcsv/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.csv/lib/talendcsv.jar" REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -81,11 +81,11 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Talen File Enhanced" MODULE="talend_file_enhanced-1.1.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="Talen File Enhanced" MODULE="talend_file_enhanced_20070724.jar" REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
<RETURNS>
|
||||
<RETURN NAME="NB_LINE" TYPE="id_Integer" AVAILABILITY="AFTER" />
|
||||
</RETURNS>
|
||||
</COMPONENT>
|
||||
</COMPONENT>
|
||||
@@ -39,7 +39,7 @@
|
||||
String skipEmptyRows = "true"; //("true").equals(ElementParameterParser.getValue(node, "__SKIP_EMPTY_ROWS__"));
|
||||
boolean trimAll = ("true").equals(ElementParameterParser.getValue(node,"__TRIMALL__"));
|
||||
String encoding = ElementParameterParser.getValue(node,"__ENCODING__");
|
||||
//need to process rows longger than 100,000 characters, the property SafetySwitch(in talend_file_enhanced-1.1.jar) should be sent to false.(the default is true)
|
||||
//need to process rows longger than 100,000 characters, the property SafetySwitch(in talend_file_enhanced_20070724.jar) should be sent to false.(the default is true)
|
||||
//that means if check the option(true), the logic value of bSafetySwitch should be changed to false (negate the property)
|
||||
boolean bSafetySwitch = !(("true").equals(ElementParameterParser.getValue(node, "__PROCESS_LONG_ROW__")));
|
||||
String advancedSeparatorStr = ElementParameterParser.getValue(node, "__ADVANCED_SEPARATOR__");
|
||||
|
||||
@@ -128,7 +128,7 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Talen File Enhanced" MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="Talen File Enhanced" MODULE="talend_file_enhanced_20070724.jar" MVN="mvn:org.talend.libraries/talend_file_enhanced_20070724/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced_20070724.jar" REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user