Compare commits
6 Commits
release/7.
...
clesaec/TD
| Author | SHA1 | Date | |
|---|---|---|---|
|
451f25526a | ||
|
|
b4c77184f3 | ||
|
|
4efe837b40 | ||
|
|
2164c692d2 | ||
|
|
c1de5f5fc0 | ||
|
|
bd31878bcd |
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.abstractmap.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.alfrescooutput.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.business.diagram.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.codegen.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.components.exchange.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.core.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.dbmap.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.documentation.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.fileoutputxml.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.gefabstractmap.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.mapper.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.maven.job.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.rowgenerator.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.runprocess.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.scd.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.webservice.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.xmlmap.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.expressionbuilder.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.presentation.onboarding.resource.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.presentation.onboarding.resource.tos.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.repository.ftp.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.repository.json.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.repository.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.repository.view.di.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.sbi.engines.client.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.spagic.engines.client.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.sqlbuilder.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.business.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.fragment.jars.di.feature</artifactId>
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
<import feature="org.talend.tos.feature" version="0.0.0" match="greaterOrEqual"/>
|
||||
</requires>
|
||||
<plugin id="org.talend.designer.maven.tos" download-size="0" install-size="0" version="0.0.0" fragment="true"/>
|
||||
<plugin id="org.talend.libraries.mdm.webservice.ce" download-size="0" install-size="0" version="0.0.0" fragment="true"/>
|
||||
<plugin id="org.talend.presentation.onboarding" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
|
||||
<plugin id="org.talend.presentation.onboarding.nl" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
|
||||
<plugin id="org.talend.presentation.onboarding.resource" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.rcp.branding.tos.feature</artifactId>
|
||||
|
||||
@@ -55,6 +55,5 @@
|
||||
<plugin id="org.talend.testutils" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
|
||||
<plugin id="org.talend.updates.runtime.test" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
|
||||
<plugin id="org.talend.utils.test" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
|
||||
<plugin id="org.talend.sdk.component.studio-integration.test" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
|
||||
<plugin id="test.all.test.suite" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
|
||||
</feature>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.rcp.branding.tos.test.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.repository.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.sbi.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.spagic.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.tos.components.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.tos.core.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.tos.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.tos.libraries.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.abstractmap</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.alfrescooutput</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.business.diagram</artifactId>
|
||||
|
||||
@@ -22,7 +22,6 @@ Require-Bundle: org.talend.core.repository,
|
||||
org.apache.log4j,
|
||||
org.talend.components.api,
|
||||
org.talend.daikon,
|
||||
org.talend.daikon.crypto.utils,
|
||||
org.talend.designer.core.generic
|
||||
Eclipse-LazyStart: true
|
||||
Bundle-ClassPath: .
|
||||
|
||||
@@ -59,76 +59,88 @@ if((codePart.equals(ECodePart.END))&&(stat || logstashCurrent)){
|
||||
List<String> needToEndConnNames = new ArrayList<String>();
|
||||
INode nextNode = node.getOutgoingConnections(EConnectionType.ITERATE).get(0).getTarget();
|
||||
NodeUtil.fillConnectionsForStat(needToEndConnNames, nextNode);
|
||||
if(!needToEndConnNames.isEmpty()) {
|
||||
if(stat && logstashCurrent) {
|
||||
%>
|
||||
runStat.updateStatAndLog(execStat,enableLogStash,iterateId,2,0<%for(String connName : needToEndConnNames){%>,"<%=connName%>"<%}%>);
|
||||
%>
|
||||
|
||||
<%if(stat) {%>
|
||||
if(execStat){
|
||||
<%
|
||||
for(String connName : needToEndConnNames){
|
||||
%>
|
||||
runStat.updateStatOnConnection("<%=connName%>"+iterateId,2, 0);
|
||||
<%
|
||||
} else {
|
||||
if(stat) {%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection(iterateId,2,0<%for(String connName : needToEndConnNames){%>,"<%=connName%>"<%}%>);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {//now only finish the log, not send, TODO%>
|
||||
if(enableLogStash){
|
||||
runStat.log(iterateId,2,0<%for(String connName : needToEndConnNames){%>,"<%=connName%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
}
|
||||
}
|
||||
|
||||
if(connSet.size()>0) {
|
||||
if(stat && logstashCurrent && (connSet.size()==1)) {//the most common case, write this ugly logic for 65535 issue
|
||||
for(IConnection con:connSet){
|
||||
INode source = con.getSource();
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
%>
|
||||
if(runStat.updateStatAndLog(execStat,enableLogStash,resourceMap,iterateId,"<%=con.getUniqueName()%>",2,0,
|
||||
<%=jobStructureCatcher.getUniqueName()%>,"<%=sourceNodeId%>","<%=sourceNodeComponent%>","<%=node.getUniqueName()%>","<%=node.getComponent().getName()%>","<%="REJECT".equals(con.getConnectorName()) ? "reject" : "output"%>")) {
|
||||
<%=jobStructureCatcher.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
}
|
||||
<%
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if(stat){
|
||||
%>
|
||||
if(execStat){
|
||||
runStat.updateStat(resourceMap,iterateId,2,0<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {//now only finish the log, not send, TODO%>
|
||||
if(enableLogStash){
|
||||
<%
|
||||
for(String connName : needToEndConnNames){
|
||||
%>
|
||||
runStat.logStatOnConnection("<%=connName%>"+iterateId,2, 0);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%
|
||||
}
|
||||
if(connSet.size()>0){
|
||||
for(IConnection con:connSet){
|
||||
INode source = con.getSource();
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
%>
|
||||
<%if(stat) {%>
|
||||
if(execStat){
|
||||
if(resourceMap.get("inIterateVComp") == null || !((Boolean)resourceMap.get("inIterateVComp"))){
|
||||
runStat.updateStatOnConnection("<%=con.getUniqueName()%>"+iterateId,2, 0);
|
||||
}
|
||||
}
|
||||
<%}%>
|
||||
|
||||
if(logstashCurrent){
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
<%
|
||||
for(IConnection con:connSet){
|
||||
INode source = con.getSource();
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash){
|
||||
if(resourceMap.get("inIterateVComp") == null || !((Boolean)resourceMap.get("inIterateVComp"))){
|
||||
<%
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
%>
|
||||
if(runStat.log(resourceMap,iterateId,"<%=con.getUniqueName()%>",2,0,
|
||||
<%=jobStructureCatcher.getUniqueName()%>,"<%=sourceNodeId%>","<%=sourceNodeComponent%>","<%=node.getUniqueName()%>","<%=node.getComponent().getName()%>","<%="REJECT".equals(con.getConnectorName()) ? "reject" : "output"%>")) {
|
||||
<%=jobStructureCatcher.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
}
|
||||
RunStat.StatBean talend_statebean = runStat.logStatOnConnection("<%=con.getUniqueName()%>"+iterateId,2, 0);
|
||||
|
||||
<%=jobStructureCatcher.getUniqueName() %>.addConnectionMessage(
|
||||
"<%=sourceNodeId%>",
|
||||
"<%=sourceNodeComponent%>",
|
||||
false,
|
||||
"<%="REJECT".equals(con.getConnectorName()) ? "reject" : "output"%>",
|
||||
"<%=con.getUniqueName()%>",
|
||||
talend_statebean.getNbLine(),
|
||||
talend_statebean.getStartTime(),
|
||||
talend_statebean.getEndTime()
|
||||
);
|
||||
|
||||
<%=jobStructureCatcher.getUniqueName() %>.addConnectionMessage(
|
||||
"<%=node.getUniqueName()%>",
|
||||
"<%=node.getComponent().getName()%>",
|
||||
true,
|
||||
"input",
|
||||
"<%=con.getUniqueName()%>",
|
||||
talend_statebean.getNbLine(),
|
||||
talend_statebean.getStartTime(),
|
||||
talend_statebean.getEndTime()
|
||||
);
|
||||
<%=jobStructureCatcher.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
<%
|
||||
break;
|
||||
}
|
||||
}
|
||||
%>
|
||||
}
|
||||
<%
|
||||
%>
|
||||
}
|
||||
}
|
||||
}
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -146,82 +146,83 @@
|
||||
boolean logstashCurrent = !cid.startsWith("tJobStructureCatcher") && !cid.startsWith("talend") && enableLogStash;
|
||||
|
||||
if ((codePart.equals(ECodePart.BEGIN))&&(stat || logstashCurrent)&&connSet.size()>0) {
|
||||
if(containsTPartitioner) {
|
||||
%>
|
||||
if(<%if(stat){%>execStat<%}%><%if(stat && logstashCurrent){%> || <%}%><%if(logstashCurrent){%>enableLogStash<%}%>) {
|
||||
runStat.updateStatOnConnectionAndLog(resourceMap,globalMap,iterateLoop,iterateId,<%if(stat){%>execStat<%} else {%>false<%}%>,enableLogStash,0<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
if(stat && logstashCurrent) {
|
||||
%>
|
||||
runStat.updateStatAndLog(execStat,enableLogStash,resourceMap,iterateId,0,0<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
<%
|
||||
} else {
|
||||
if(stat) {
|
||||
%>
|
||||
if(execStat) {
|
||||
runStat.updateStatOnConnection(resourceMap,iterateId,0,0<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
for(IConnection con:connSet){
|
||||
%>
|
||||
if (<%if(stat){%>execStat<%}%><%if(stat && logstashCurrent){%> || <%}%><%if(logstashCurrent){%>enableLogStash<%}%>) {
|
||||
if(resourceMap.get("inIterateVComp") == null){
|
||||
<%if(containsTPartitioner){%>
|
||||
java.util.concurrent.ConcurrentHashMap<Object, Object> concurrentHashMap_<%=con.getUniqueName() %> = (java.util.concurrent.ConcurrentHashMap) globalMap.get("concurrentHashMap");
|
||||
concurrentHashMap_<%=con.getUniqueName() %>.putIfAbsent("<%=con.getUniqueName() %>" + iterateLoop,new java.util.concurrent.atomic.AtomicInteger(0));
|
||||
java.util.concurrent.atomic.AtomicInteger stats_<%=con.getUniqueName() %> = (java.util.concurrent.atomic.AtomicInteger) concurrentHashMap_<%=con.getUniqueName() %>.get("<%=con.getUniqueName() %>" + iterateLoop);
|
||||
int step_<%=con.getUniqueName() %> = stats_<%=con.getUniqueName() %>.incrementAndGet()<=1?0:1;
|
||||
<%if(stat) {%>
|
||||
if(execStat) {
|
||||
runStat.updateStatOnConnection("<%=con.getUniqueName() %>"+iterateId, step_<%=con.getUniqueName()%>, 0);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.logStatOnConnection("<%=con.getUniqueName()%>"+iterateId, step_<%=con.getUniqueName()%>, 0);
|
||||
}
|
||||
<%}%>
|
||||
<%}else{%>
|
||||
<%if(stat) {%>
|
||||
if(execStat) {
|
||||
runStat.updateStatOnConnection("<%=con.getUniqueName() %>"+iterateId, 0, 0);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.logStatOnConnection("<%=con.getUniqueName()%>"+iterateId, 0, 0);
|
||||
}
|
||||
<%}%>
|
||||
<%}%>
|
||||
}
|
||||
|
||||
if(logstashCurrent) {
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
runStat.log(resourceMap,iterateId,0,0<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
if((codePart.equals(ECodePart.MAIN))&&(stat || logstashCurrent)&&connSet.size()>0){
|
||||
if(!node.getComponent().useMerge()) {
|
||||
if(stat && logstashCurrent) {
|
||||
%>
|
||||
runStat.updateStatAndLog(execStat,enableLogStash,iterateId,1,1<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
<%
|
||||
} else {
|
||||
if(stat) {
|
||||
%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection(iterateId,1,1<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
for(IConnection con:connSet){
|
||||
%>
|
||||
|
||||
//<%=con.getUniqueName()%>
|
||||
//<%=(String)codeGenArgument.getIncomingName()%>
|
||||
|
||||
|
||||
<%if (!node.getComponent().useMerge()) {%>
|
||||
<%if(stat) {%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection("<%=con.getUniqueName() %>"+iterateId,1, 1);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
if(logstashCurrent) {
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
runStat.log(iterateId,1,1<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.logStatOnConnection("<%=con.getUniqueName() %>"+iterateId,1, 1);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for(IConnection connection:connSet){
|
||||
if(connection.getUniqueName().equals((String)codeGenArgument.getIncomingName())){
|
||||
if(stat && logstashCurrent) {
|
||||
%>
|
||||
runStat.updateStatAndLog(execStat,enableLogStash,iterateId,1,1<%for(IConnection con : connSet){if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){%>,"<%=con.getUniqueName()%>"<%}}%>);
|
||||
<%
|
||||
} else {
|
||||
if(stat) {%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection(iterateId,1,1<%for(IConnection con : connSet){if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){%>,"<%=con.getUniqueName()%>"<%}}%>);
|
||||
}
|
||||
<%}
|
||||
|
||||
if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.log(iterateId,1,1<%for(IConnection con : connSet){if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){%>,"<%=con.getUniqueName()%>"<%}}%>);
|
||||
}
|
||||
<%}
|
||||
}
|
||||
<%}%>
|
||||
<%
|
||||
} else if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){
|
||||
%>
|
||||
<%if(stat) {%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection("<%=con.getUniqueName() %>"+iterateId,1, 1);
|
||||
}
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.logStatOnConnection("<%=con.getUniqueName() %>"+iterateId,1, 1);
|
||||
}
|
||||
<%}%>
|
||||
<%}%>
|
||||
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
@@ -250,7 +251,7 @@
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
<%=jobStructureCatcher.getUniqueName() %>.addCM("<%=node.getUniqueName()%>", "<%=node.getComponent().getName()%>");
|
||||
<%=jobStructureCatcher.getUniqueName() %>.addComponentMessage("<%=node.getUniqueName()%>", "<%=node.getComponent().getName()%>");
|
||||
<%=jobStructureCatcher.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
}
|
||||
<%
|
||||
@@ -265,37 +266,46 @@
|
||||
List<String> needToStartConnNames = new ArrayList<String>();
|
||||
INode nextNode = node.getOutgoingConnections(EConnectionType.ITERATE).get(0).getTarget();
|
||||
NodeUtil.fillConnectionsForStat(needToStartConnNames, nextNode);
|
||||
if(needToStartConnNames.isEmpty()) {
|
||||
//do nothing
|
||||
} else if(containsTPartitioner){
|
||||
%>
|
||||
if(<%if(stat){%>execStat<%}%><%if(stat && logstashCurrent){%> || <%}%><%if(logstashCurrent){%>enableLogStash<%}%>){
|
||||
runStat.updateStatOnConnectionAndLog(globalMap,iterateLoop,iterateId,<%if(stat){%>execStat<%} else {%>false<%}%>,enableLogStash,0<%for(String connName : needToStartConnNames){%>,"<%=connName%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
if(stat && logstashCurrent) {
|
||||
%>
|
||||
runStat.updateStatAndLog(execStat,enableLogStash,iterateId,0,0<%for(String connName : needToStartConnNames){%>,"<%=connName%>"<%}%>);
|
||||
<%
|
||||
} else {
|
||||
if(stat){
|
||||
%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection(iterateId,0,0<%for(String connName : needToStartConnNames){%>,"<%=connName%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash){
|
||||
runStat.log(iterateId,0,0<%for(String connName : needToStartConnNames){%>,"<%=connName%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
for(String connName : needToStartConnNames){
|
||||
%>
|
||||
<%if(containsTPartitioner){%>
|
||||
java.util.concurrent.ConcurrentHashMap<Object, Object> concurrentHashMap_<%=connName%> = (java.util.concurrent.ConcurrentHashMap) globalMap.get("concurrentHashMap");
|
||||
concurrentHashMap_<%=connName%>.putIfAbsent("<%=connName%>" + iterateLoop,new java.util.concurrent.atomic.AtomicInteger(0));
|
||||
java.util.concurrent.atomic.AtomicInteger stats_<%=connName%> = (java.util.concurrent.atomic.AtomicInteger) concurrentHashMap_<%=connName%>.get("<%=connName%>" + iterateLoop);
|
||||
|
||||
int step_<%=connName %> = stats_<%=connName%>.incrementAndGet()<=1?0:1;
|
||||
<%if(stat) {%>
|
||||
if(execStat) {
|
||||
runStat.updateStatOnConnection("<%=connName%>"+iterateId, step_<%=connName%>, 0);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.logStatOnConnection("<%=connName%>"+iterateId, step_<%=connName%>, 0);
|
||||
}
|
||||
<%}%>
|
||||
<%}else{%>
|
||||
<%if(stat) {%>
|
||||
if(execStat) {
|
||||
runStat.updateStatOnConnection("<%=connName%>"+iterateId, 0, 0);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.logStatOnConnection("<%=connName%>"+iterateId, 0, 0);
|
||||
}
|
||||
<%}%>
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
<%
|
||||
}else if(codePart.equals(ECodePart.MAIN)){
|
||||
%>
|
||||
resourceMap.put("inIterateVComp", true);
|
||||
|
||||
@@ -445,7 +445,6 @@
|
||||
lastStr = "";
|
||||
}
|
||||
}
|
||||
enableLogStash = "true".equalsIgnoreCase(System.getProperty("monitoring"));
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(!"".equals(log4jLevel)){
|
||||
@@ -494,11 +493,7 @@
|
||||
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("appender.file.maxsize", "52428800");
|
||||
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("appender.file.maxbackup", "20");
|
||||
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("host", "false");
|
||||
|
||||
System.getProperties().stringPropertyNames().stream()
|
||||
.filter(it -> it.startsWith("monitoring.audit.logger.properties."))
|
||||
.forEach(key -> properties_<%=jobCatcherNode.getUniqueName()%>.setProperty(key.substring("monitoring.audit.logger.properties.".length()), System.getProperty(key)));
|
||||
|
||||
|
||||
auditLogger_<%=jobCatcherNode.getUniqueName()%> = org.talend.job.audit.JobEventAuditLoggerFactory.createJobAuditLogger(properties_<%=jobCatcherNode.getUniqueName()%>);
|
||||
}
|
||||
<%
|
||||
@@ -1514,10 +1509,12 @@ if (execStat) {
|
||||
}
|
||||
} else if (arg.startsWith("--log4jLevel=")) {
|
||||
log4jLevel = arg.substring(13);
|
||||
} else if (arg.startsWith("--monitoring") && arg.contains("=")) {//for trunjob call
|
||||
final int equal = arg.indexOf('=');
|
||||
final String key = arg.substring("--".length(), equal);
|
||||
System.setProperty(key, arg.substring(equal + 1));
|
||||
} else if (arg.startsWith("--monitoring=")) {//for trunjob call
|
||||
enableLogStash = "true".equalsIgnoreCase(arg.substring(13));
|
||||
}
|
||||
|
||||
if(!enableLogStash) {
|
||||
enableLogStash = "true".equalsIgnoreCase(System.getProperty("monitoring"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -69,140 +69,145 @@ if(hasInput){
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
boolean hasValidInput = inputConn!=null;
|
||||
if (hasValidInput) {
|
||||
|
||||
IMetadataTable metadata = null;
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
boolean haveValidNodeMetadata = ((metadatas != null) && (metadatas.size() > 0) && (metadata = metadatas.get(0)) != null);
|
||||
|
||||
if (hasValidInput && haveValidNodeMetadata) {
|
||||
List<IMetadataColumn> input_columnList = inputConn.getMetadataTable().getListColumns();
|
||||
if(input_columnList == null) {
|
||||
input_columnList = new ArrayList<IMetadataColumn>();
|
||||
}
|
||||
// add incoming (not present) columns to enforcer for this comps
|
||||
if (cid.contains("tDataStewardship") || cid.contains("tMarkLogic")){
|
||||
%>
|
||||
boolean shouldCreateRuntimeSchemaForIncomingNode = false;
|
||||
<%
|
||||
for (int i = 0; i < input_columnList.size(); i++) {
|
||||
if(!input_columnList.get(i).getTalendType().equals("id_Dynamic")) {
|
||||
|
||||
if (input_columnList!=null && !input_columnList.isEmpty()) {
|
||||
// add incoming (not present) columns to enforcer for this comps
|
||||
if (cid.contains("tDataStewardship") || cid.contains("tMarkLogic")){
|
||||
%>
|
||||
if (incomingEnforcer_<%=cid%>.getDesignSchema().getField("<%=input_columnList.get(i)%>") == null){
|
||||
incomingEnforcer_<%=cid%>.addIncomingNodeField("<%=input_columnList.get(i)%>", ((Object) <%=inputConn.getName()%>.<%=input_columnList.get(i)%>).getClass().getCanonicalName());
|
||||
shouldCreateRuntimeSchemaForIncomingNode = true;
|
||||
boolean shouldCreateRuntimeSchemaForIncomingNode = false;
|
||||
<%
|
||||
for (int i = 0; i < input_columnList.size(); i++) {
|
||||
if(!input_columnList.get(i).getTalendType().equals("id_Dynamic")) {
|
||||
%>
|
||||
if (incomingEnforcer_<%=cid%>.getDesignSchema().getField("<%=input_columnList.get(i)%>") == null){
|
||||
incomingEnforcer_<%=cid%>.addIncomingNodeField("<%=input_columnList.get(i)%>", ((Object) <%=inputConn.getName()%>.<%=input_columnList.get(i)%>).getClass().getCanonicalName());
|
||||
shouldCreateRuntimeSchemaForIncomingNode = true;
|
||||
}
|
||||
<%
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
if (shouldCreateRuntimeSchemaForIncomingNode){
|
||||
incomingEnforcer_<%=cid%>.createRuntimeSchema();
|
||||
}
|
||||
<%
|
||||
}
|
||||
// If there are dynamic columns in the schema, they need to be
|
||||
// initialized into the runtime schema of the actual IndexedRecord
|
||||
// provided to the component.
|
||||
|
||||
int dynamicPos = -1;
|
||||
for (int i = 0; i < input_columnList.size(); i++) {
|
||||
if (input_columnList.get(i).getTalendType().equals("id_Dynamic")) {
|
||||
dynamicPos = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (dynamicPos != -1) {
|
||||
%>
|
||||
if (!incomingEnforcer_<%=cid%>.areDynamicFieldsInitialized()) {
|
||||
// Initialize the dynamic columns when they are first encountered.
|
||||
for (routines.system.DynamicMetadata dm_<%=cid%> : <%=inputConn.getName()%>.<%=input_columnList.get(dynamicPos).getLabel()%>.metadatas) {
|
||||
incomingEnforcer_<%=cid%>.addDynamicField(
|
||||
dm_<%=cid%>.getName(),
|
||||
dm_<%=cid%>.getType(),
|
||||
dm_<%=cid%>.getLogicalType(),
|
||||
dm_<%=cid%>.getFormat(),
|
||||
dm_<%=cid%>.getDescription(),
|
||||
dm_<%=cid%>.isNullable());
|
||||
}
|
||||
incomingEnforcer_<%=cid%>.createRuntimeSchema();
|
||||
}
|
||||
<%
|
||||
}
|
||||
|
||||
%>
|
||||
incomingEnforcer_<%=cid%>.createNewRecord();
|
||||
<%
|
||||
for (int i = 0; i < input_columnList.size(); i++) { // column
|
||||
IMetadataColumn column = input_columnList.get(i);
|
||||
if (dynamicPos != i) {
|
||||
%>
|
||||
//skip the put action if the input column doesn't appear in component runtime schema
|
||||
if (incomingEnforcer_<%=cid%>.getRuntimeSchema().getField("<%=input_columnList.get(i)%>") != null){
|
||||
incomingEnforcer_<%=cid%>.put("<%=column.getLabel()%>", <%=inputConn.getName()%>.<%=column.getLabel()%>);
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
for (int i = 0; i < <%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnCount(); i++) {
|
||||
incomingEnforcer_<%=cid%>.put(<%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnMetadata(i).getName(),
|
||||
<%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnValue(i));
|
||||
if (shouldCreateRuntimeSchemaForIncomingNode){
|
||||
incomingEnforcer_<%=cid%>.createRuntimeSchema();
|
||||
}
|
||||
<%
|
||||
}
|
||||
} // column
|
||||
// If there are dynamic columns in the schema, they need to be
|
||||
// initialized into the runtime schema of the actual IndexedRecord
|
||||
// provided to the component.
|
||||
|
||||
// If necesary, generate the code to handle outgoing connections.
|
||||
// TODO: For now, this can only handle one outgoing record for
|
||||
// each incoming record. To handle multiple outgoing records, code
|
||||
// generation needs to occur in component_begin in order to open
|
||||
// a for() loop.
|
||||
int dynamicPos = -1;
|
||||
for (int i = 0; i < input_columnList.size(); i++) {
|
||||
if (input_columnList.get(i).getTalendType().equals("id_Dynamic")) {
|
||||
dynamicPos = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// There will be a ClassCastException if the output component does
|
||||
// not implement WriterWithFeedback, but permits outgoing
|
||||
// connections.
|
||||
|
||||
ComponentProperties componentProps = node.getComponentProperties();
|
||||
ProcessPropertiesGenerator generator = new ProcessPropertiesGenerator(cid, component);
|
||||
List<Component.CodegenPropInfo> propsToProcess = component.getCodegenPropInfos(componentProps);
|
||||
for (Component.CodegenPropInfo propInfo : propsToProcess) { // propInfo
|
||||
List<NamedThing> properties = propInfo.props.getProperties();
|
||||
for (NamedThing prop : properties) { // property
|
||||
if (prop instanceof Property) { // if, only deal with valued Properties
|
||||
Property property = (Property)prop;
|
||||
if (property.getFlags() != null && (property.getFlags().contains(Property.Flags.DESIGN_TIME_ONLY) || property.getFlags().contains(Property.Flags.HIDDEN)))
|
||||
continue;
|
||||
if(property.getTaggedValue(IGenericConstants.DYNAMIC_PROPERTY_VALUE)!=null && Boolean.valueOf(String.valueOf(property.getTaggedValue(IGenericConstants.DYNAMIC_PROPERTY_VALUE)))) {
|
||||
generator.setPropertyValues(property, propInfo, null, false, false);
|
||||
if (dynamicPos != -1) {
|
||||
%>
|
||||
if (!incomingEnforcer_<%=cid%>.areDynamicFieldsInitialized()) {
|
||||
// Initialize the dynamic columns when they are first encountered.
|
||||
for (routines.system.DynamicMetadata dm_<%=cid%> : <%=inputConn.getName()%>.<%=input_columnList.get(dynamicPos).getLabel()%>.metadatas) {
|
||||
incomingEnforcer_<%=cid%>.addDynamicField(
|
||||
dm_<%=cid%>.getName(),
|
||||
dm_<%=cid%>.getType(),
|
||||
dm_<%=cid%>.getLogicalType(),
|
||||
dm_<%=cid%>.getFormat(),
|
||||
dm_<%=cid%>.getDescription(),
|
||||
dm_<%=cid%>.isNullable());
|
||||
}
|
||||
incomingEnforcer_<%=cid%>.createRuntimeSchema();
|
||||
}
|
||||
<%
|
||||
}
|
||||
|
||||
%>
|
||||
incomingEnforcer_<%=cid%>.createNewRecord();
|
||||
<%
|
||||
for (int i = 0; i < input_columnList.size(); i++) { // column
|
||||
IMetadataColumn column = input_columnList.get(i);
|
||||
if (dynamicPos != i) {
|
||||
%>
|
||||
//skip the put action if the input column doesn't appear in component runtime schema
|
||||
if (incomingEnforcer_<%=cid%>.getRuntimeSchema().getField("<%=input_columnList.get(i)%>") != null){
|
||||
incomingEnforcer_<%=cid%>.put("<%=column.getLabel()%>", <%=inputConn.getName()%>.<%=column.getLabel()%>);
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
for (int i = 0; i < <%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnCount(); i++) {
|
||||
incomingEnforcer_<%=cid%>.put(<%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnMetadata(i).getName(),
|
||||
<%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnValue(i));
|
||||
}
|
||||
<%
|
||||
}
|
||||
} // column
|
||||
|
||||
// If necesary, generate the code to handle outgoing connections.
|
||||
// TODO: For now, this can only handle one outgoing record for
|
||||
// each incoming record. To handle multiple outgoing records, code
|
||||
// generation needs to occur in component_begin in order to open
|
||||
// a for() loop.
|
||||
|
||||
// There will be a ClassCastException if the output component does
|
||||
// not implement WriterWithFeedback, but permits outgoing
|
||||
// connections.
|
||||
|
||||
ComponentProperties componentProps = node.getComponentProperties();
|
||||
ProcessPropertiesGenerator generator = new ProcessPropertiesGenerator(cid, component);
|
||||
List<Component.CodegenPropInfo> propsToProcess = component.getCodegenPropInfos(componentProps);
|
||||
for (Component.CodegenPropInfo propInfo : propsToProcess) { // propInfo
|
||||
List<NamedThing> properties = propInfo.props.getProperties();
|
||||
for (NamedThing prop : properties) { // property
|
||||
if (prop instanceof Property) { // if, only deal with valued Properties
|
||||
Property property = (Property)prop;
|
||||
if (property.getFlags() != null && (property.getFlags().contains(Property.Flags.DESIGN_TIME_ONLY) || property.getFlags().contains(Property.Flags.HIDDEN)))
|
||||
continue;
|
||||
if(property.getTaggedValue(IGenericConstants.DYNAMIC_PROPERTY_VALUE)!=null && Boolean.valueOf(String.valueOf(property.getTaggedValue(IGenericConstants.DYNAMIC_PROPERTY_VALUE)))) {
|
||||
generator.setPropertyValues(property, propInfo, null, false, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
} // property
|
||||
} // propInfo
|
||||
|
||||
%>
|
||||
org.apache.avro.generic.IndexedRecord data_<%=cid%> = incomingEnforcer_<%=cid%>.getCurrentRecord();
|
||||
|
||||
<%
|
||||
boolean isParallelize ="true".equalsIgnoreCase(ElementParameterParser.getValue(node, "__PARALLELIZE__"));
|
||||
if (isParallelize) {
|
||||
String sourceComponentId = inputConn.getSource().getUniqueName();
|
||||
if(sourceComponentId!=null && sourceComponentId.contains("tAsyncIn")) {
|
||||
%>
|
||||
globalMap.put(buffersSizeKey_<%=cid%>, buffersSize_<%=sourceComponentId%>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
writer_<%=cid%>.write(data_<%=cid%>);
|
||||
|
||||
nb_line_<%=cid %>++;
|
||||
<%if(hasMainOutput){
|
||||
} // property
|
||||
} // propInfo
|
||||
|
||||
%>
|
||||
if(!(writer_<%=cid%> instanceof org.talend.components.api.component.runtime.WriterWithFeedback)) {
|
||||
// For no feedback writer,just pass the input record to the output
|
||||
if (data_<%=cid%>!=null) {
|
||||
outgoingMainRecordsList_<%=cid%> = java.util.Arrays.asList(data_<%=cid%>);
|
||||
}
|
||||
}
|
||||
<%
|
||||
org.apache.avro.generic.IndexedRecord data_<%=cid%> = incomingEnforcer_<%=cid%>.getCurrentRecord();
|
||||
|
||||
<%
|
||||
boolean isParallelize ="true".equalsIgnoreCase(ElementParameterParser.getValue(node, "__PARALLELIZE__"));
|
||||
if (isParallelize) {
|
||||
String sourceComponentId = inputConn.getSource().getUniqueName();
|
||||
if(sourceComponentId!=null && sourceComponentId.contains("tAsyncIn")) {
|
||||
%>
|
||||
globalMap.put(buffersSizeKey_<%=cid%>, buffersSize_<%=sourceComponentId%>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
writer_<%=cid%>.write(data_<%=cid%>);
|
||||
|
||||
nb_line_<%=cid %>++;
|
||||
<%if(hasMainOutput){
|
||||
%>
|
||||
if(!(writer_<%=cid%> instanceof org.talend.components.api.component.runtime.WriterWithFeedback)) {
|
||||
// For no feedback writer,just pass the input record to the output
|
||||
if (data_<%=cid%>!=null) {
|
||||
outgoingMainRecordsList_<%=cid%> = java.util.Arrays.asList(data_<%=cid%>);
|
||||
}
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
} // canStart
|
||||
|
||||
@@ -139,7 +139,7 @@ for (INode node : process.getNodesOfType("tRESTClient")) {
|
||||
}
|
||||
|
||||
boolean talendEsbJobFactory = actAsProvider || !process.getNodesOfType("tRouteInput").isEmpty();
|
||||
boolean talendEsbJob = talendEsbJobFactory || actAsConsumer || ProcessorUtilities.isEsbJob(process);
|
||||
boolean talendEsbJob = talendEsbJobFactory || actAsConsumer || ProcessorUtilities.isEsbJob(process.getId(), process.getVersion());
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(process, "__LOG4J_ACTIVATE__"));
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.codegen</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.components.exchange</artifactId>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>checkArchive-1.1-20190917</artifactId>
|
||||
<artifactId>checkArchive-1.1-20181130</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<name>checkArchive</name>
|
||||
<description>Dependence for tFileArchive and tFileUnAchive</description>
|
||||
@@ -41,7 +41,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<version>1.19</version>
|
||||
<version>1.10</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>commons-net-ftps-proxy</artifactId>
|
||||
<version>3.6.1-talend-20190819</version>
|
||||
<version>3.6.1-talend-20190128</version>
|
||||
|
||||
<name>commons-net-talend</name>
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ package org.talend.ftp;
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Method;
|
||||
import java.net.Socket;
|
||||
import java.net.InetAddress;
|
||||
import java.util.Locale;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
@@ -30,13 +29,11 @@ public class SSLSessionReuseFTPSClient extends FTPSClient {
|
||||
final Object cache = sessionHostPortCache.get(context);
|
||||
final Method putMethod = cache.getClass().getDeclaredMethod("put", Object.class, Object.class);
|
||||
putMethod.setAccessible(true);
|
||||
InetAddress address = socket.getInetAddress();
|
||||
int port = socket.getPort();
|
||||
|
||||
String key = String.format("%s:%s", address.getHostName(), String.valueOf(port)).toLowerCase(Locale.ROOT);
|
||||
putMethod.invoke(cache, key, session);
|
||||
|
||||
key = String.format("%s:%s", address.getHostAddress(), String.valueOf(port)).toLowerCase(Locale.ROOT);
|
||||
final Method getHostMethod = socket.getClass().getDeclaredMethod("getHost");
|
||||
getHostMethod.setAccessible(true);
|
||||
Object host = getHostMethod.invoke(socket);
|
||||
final String key =
|
||||
String.format("%s:%s", host, String.valueOf(socket.getPort())).toLowerCase(Locale.ROOT);
|
||||
putMethod.invoke(cache, key, session);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<groupId>net.sf.json-lib</groupId>
|
||||
<artifactId>json-lib</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<version>2.4.3-talend</version>
|
||||
<version>2.4.1-talend</version>
|
||||
<name>json-lib</name>
|
||||
|
||||
<properties>
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
*/
|
||||
package net.sf.json.util;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import net.sf.json.JSONArray;
|
||||
import net.sf.json.JSONException;
|
||||
import net.sf.json.JSONNull;
|
||||
@@ -415,7 +414,7 @@ public class JSONTokener {
|
||||
}
|
||||
|
||||
try{
|
||||
return createNumber(s);
|
||||
return NumberUtils.createNumber(s);
|
||||
}catch( Exception e ){
|
||||
return s;
|
||||
}
|
||||
@@ -436,26 +435,6 @@ public class JSONTokener {
|
||||
return s;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method has been added to fix https://jira.talendforge.org/browse/TDI-42689
|
||||
*
|
||||
* @param s The String representation of the number
|
||||
* @return The Number instance
|
||||
*/
|
||||
private Number createNumber(String s){
|
||||
boolean isDecimal = s.indexOf('.') != -1;
|
||||
|
||||
if(isDecimal){
|
||||
Double d = Double.valueOf(s);
|
||||
if(Double.POSITIVE_INFINITY == Math.abs(d)){
|
||||
return new BigDecimal(s);
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
return NumberUtils.createNumber(s);
|
||||
}
|
||||
|
||||
/**
|
||||
* Look at the next character in the source string.
|
||||
*
|
||||
|
||||
@@ -184,12 +184,14 @@ public final class JSONUtils {
|
||||
return Integer.class;
|
||||
}else if( isLong( n ) ){
|
||||
return Long.class;
|
||||
}else if( isFloat( n ) ){
|
||||
return Float.class;
|
||||
}else if( isBigInteger( n ) ){
|
||||
return BigInteger.class;
|
||||
}else if( isDouble( n ) ){
|
||||
return Double.class;
|
||||
}else if( isBigDecimal( n ) ){
|
||||
return BigDecimal.class;
|
||||
}else if( isDouble( n ) ){
|
||||
return Double.class;
|
||||
}else{
|
||||
throw new JSONException( "Unsupported type" );
|
||||
}
|
||||
|
||||
@@ -163,11 +163,6 @@ public class XMLSerializer {
|
||||
*/
|
||||
private boolean useLongDecimals;
|
||||
|
||||
/**
|
||||
* flag for if parse empty elements as empty strings
|
||||
*/
|
||||
private boolean useEmptyStrings;
|
||||
|
||||
/**
|
||||
* Creates a new XMLSerializer with default options.<br>
|
||||
* <ul>
|
||||
@@ -808,8 +803,6 @@ public class XMLSerializer {
|
||||
clazz = JSONTypes.OBJECT;
|
||||
}else if( JSONTypes.ARRAY.compareToIgnoreCase( clazzText ) == 0 ){
|
||||
clazz = JSONTypes.ARRAY;
|
||||
} else if(JSONTypes.STRING.equalsIgnoreCase(clazzText)) {
|
||||
clazz = JSONTypes.STRING;
|
||||
}
|
||||
}
|
||||
return clazz;
|
||||
@@ -1421,8 +1414,6 @@ public class XMLSerializer {
|
||||
String text = element.getValue();
|
||||
params = StringUtils.split( paramsAttribute.getValue(), "," );
|
||||
setOrAccumulate( jsonObject, key, new JSONFunction( params, text ) );
|
||||
} else if( useEmptyStrings && clazz != null && clazz.equalsIgnoreCase(JSONTypes.STRING) ) {
|
||||
setTextValue(jsonObject, key, element);
|
||||
}else{
|
||||
if( isArray( element, false ) ){
|
||||
setOrAccumulate( jsonObject, key, processArrayElement( element, defaultType ) );
|
||||
@@ -1430,23 +1421,19 @@ public class XMLSerializer {
|
||||
setOrAccumulate( jsonObject, key, simplifyValue( jsonObject,
|
||||
processObjectElement( element, defaultType ) ) );
|
||||
}else{
|
||||
setTextValue(jsonObject, key, element);
|
||||
String value;
|
||||
if( isKeepCData && isCData( element ) ){
|
||||
value = "<![CDATA[" + element.getValue() + "]]>";
|
||||
}else{
|
||||
value = element.getValue();
|
||||
}
|
||||
setOrAccumulate( jsonObject, key, trimSpaceFromValue( value ) );
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void setTextValue(final JSONObject jsonObject, final String key, final Element element) {
|
||||
String value;
|
||||
if( isKeepCData && isCData( element ) ){
|
||||
value = "<![CDATA[" + element.getValue() + "]]>";
|
||||
}else{
|
||||
value = element.getValue();
|
||||
}
|
||||
setOrAccumulate( jsonObject, key, trimSpaceFromValue( value ) );
|
||||
}
|
||||
|
||||
private boolean isCData( Element element ) {
|
||||
if( element.getChildCount() == 1 ){
|
||||
final Node child = element.getChild( 0 );
|
||||
@@ -1506,14 +1493,6 @@ public class XMLSerializer {
|
||||
return str;
|
||||
}
|
||||
|
||||
public void setUseEmptyStrings(boolean useEmptyStrings) {
|
||||
this.useEmptyStrings = useEmptyStrings;
|
||||
}
|
||||
|
||||
public boolean isUseEmptyStrings() {
|
||||
return this.useEmptyStrings;
|
||||
}
|
||||
|
||||
private static class CustomElement extends Element {
|
||||
private static String getName( String name ) {
|
||||
int colon = name.indexOf( ':' );
|
||||
|
||||
@@ -27,7 +27,6 @@ import net.sf.json.processors.PropertyNameProcessor;
|
||||
import net.sf.json.sample.BeanA;
|
||||
import net.sf.json.sample.BeanB;
|
||||
import net.sf.json.sample.BeanC;
|
||||
import net.sf.json.sample.BeanD;
|
||||
import net.sf.json.sample.BeanFoo;
|
||||
import net.sf.json.sample.BeanWithFunc;
|
||||
import net.sf.json.sample.ChildBean;
|
||||
@@ -1043,17 +1042,6 @@ public class TestJSONObject extends TestCase {
|
||||
JSONArray.toArray( jsonObject.getJSONArray( "intarray" ) ) );
|
||||
}
|
||||
|
||||
public void testToBean_BeanD() {
|
||||
String json = "{bool:true,integer:1,string:\"json\",doublearray:[4.2424245783E7, 123456789.2424245783E7, 6.0]}";
|
||||
JSONObject jsonObject = JSONObject.fromObject( json );
|
||||
BeanD bean = (BeanD) JSONObject.toBean( jsonObject, BeanD.class );
|
||||
assertEquals( jsonObject.get( "bool" ), Boolean.valueOf( bean.isBool() ) );
|
||||
assertEquals( jsonObject.get( "integer" ), new Integer( bean.getInteger() ) );
|
||||
assertEquals( jsonObject.get( "string" ), bean.getString() );
|
||||
Assertions.assertEquals( bean.getDoublearray(),
|
||||
JSONArray.toArray( jsonObject.getJSONArray( "doublearray" ) ) );
|
||||
}
|
||||
|
||||
public void testToBean_ClassBean() {
|
||||
JSONObject json = new JSONObject();
|
||||
json.element( "klass", "java.lang.Object" );
|
||||
@@ -1062,29 +1050,9 @@ public class TestJSONObject extends TestCase {
|
||||
assertEquals( Object.class, bean.getKlass() );
|
||||
}
|
||||
|
||||
public void testToBean_DynaBean__BigInteger_Double() {
|
||||
BigInteger l = new BigDecimal( "1.7976931348623157E308" ).toBigInteger();
|
||||
BigDecimal m = new BigDecimal( "1.7976931348623157E307" ).add( new BigDecimal( "0.0001" ) );
|
||||
JSONObject json = new JSONObject().element( "i", BigInteger.ZERO )
|
||||
.element( "d", MorphUtils.BIGDECIMAL_ONE )
|
||||
.element( "bi", l )
|
||||
.element( "bd", m );
|
||||
Object bean = JSONObject.toBean( json );
|
||||
Object i = ((MorphDynaBean) bean).get( "i" );
|
||||
Object d = ((MorphDynaBean) bean).get( "d" );
|
||||
assertTrue( i instanceof Integer );
|
||||
assertTrue( d instanceof Integer );
|
||||
|
||||
Object bi = ((MorphDynaBean) bean).get( "bi" );
|
||||
Object bd = ((MorphDynaBean) bean).get( "bd" );
|
||||
assertTrue( bi instanceof BigInteger );
|
||||
assertTrue( bd instanceof Double );
|
||||
}
|
||||
|
||||
|
||||
public void testToBean_DynaBean__BigInteger_BigDecimal() {
|
||||
BigInteger l = new BigDecimal( "1.7976931348623157E308" ).toBigInteger();
|
||||
BigDecimal m = new BigDecimal( "-1.7976931348623157E309" ).add( new BigDecimal( "0.0001" ) );
|
||||
BigDecimal m = new BigDecimal( "1.7976931348623157E307" ).add( new BigDecimal( "0.0001" ) );
|
||||
JSONObject json = new JSONObject().element( "i", BigInteger.ZERO )
|
||||
.element( "d", MorphUtils.BIGDECIMAL_ONE )
|
||||
.element( "bi", l )
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
/*
|
||||
* Copyright 2002-2009 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package net.sf.json.sample;
|
||||
|
||||
import org.apache.commons.lang.builder.ToStringBuilder;
|
||||
import org.apache.commons.lang.builder.ToStringStyle;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
|
||||
/**
|
||||
* @author Andres Almiray <aalmiray@users.sourceforge.net>
|
||||
*/
|
||||
public class BeanD extends BeanA {
|
||||
private Double[] doublearray = new Double[3];
|
||||
|
||||
public Double[] getDoublearray() {
|
||||
return doublearray;
|
||||
}
|
||||
|
||||
public void setDoublearray(Double[] doublearray) {
|
||||
this.doublearray = doublearray;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return ToStringBuilder.reflectionToString( this, ToStringStyle.MULTI_LINE_STYLE );
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,6 @@
|
||||
package net.sf.json.util;
|
||||
|
||||
import java.io.StringWriter;
|
||||
import java.math.BigDecimal;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
import net.sf.json.JSONFunction;
|
||||
@@ -94,7 +93,7 @@ public class TestJSONBuilder extends TestCase {
|
||||
.endObject();
|
||||
JSONObject jsonObj = JSONObject.fromObject( w.toString() );
|
||||
assertEquals( Boolean.TRUE, jsonObj.get( "bool" ) );
|
||||
assertEquals( Double.valueOf( "1.1" ), jsonObj.get( "numDouble" ) );
|
||||
assertEquals( new Double( 1.1d ), jsonObj.get( "numDouble" ) );
|
||||
assertEquals( new Long( 2 ).longValue(), ((Number) jsonObj.get( "numInt" )).longValue() );
|
||||
assertEquals( "text", jsonObj.get( "text" ) );
|
||||
assertTrue( JSONUtils.isFunction( jsonObj.get( "func" ) ) );
|
||||
|
||||
@@ -20,8 +20,6 @@ import junit.framework.TestCase;
|
||||
import net.sf.json.JSONFunction;
|
||||
import net.sf.json.JSONObject;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
|
||||
/**
|
||||
* @author Andres Almiray <aalmiray@users.sourceforge.net>
|
||||
*/
|
||||
@@ -87,7 +85,7 @@ public class TestJSONStringer extends TestCase {
|
||||
.endObject();
|
||||
JSONObject jsonObj = JSONObject.fromObject( b.toString() );
|
||||
assertEquals( Boolean.TRUE, jsonObj.get( "bool" ) );
|
||||
assertEquals( Double.valueOf( "1.1" ), jsonObj.get( "numDouble" ) );
|
||||
assertEquals( new Double( 1.1d ), jsonObj.get( "numDouble" ) );
|
||||
assertEquals( new Long( 2 ).longValue(), ((Number) jsonObj.get( "numInt" )).longValue() );
|
||||
assertEquals( "text", jsonObj.get( "text" ) );
|
||||
assertTrue( JSONUtils.isFunction( jsonObj.get( "func" ) ) );
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>simpleexcel-2.2-20190722</artifactId>
|
||||
<artifactId>simpleexcel-2.1-20190507</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
|
||||
@@ -12,21 +12,17 @@
|
||||
// ============================================================================
|
||||
package com.talend.excel.xssf.event;
|
||||
|
||||
import org.apache.poi.ooxml.util.PackageHelper;
|
||||
import org.apache.poi.openxml4j.opc.OPCPackage;
|
||||
import org.apache.poi.poifs.crypt.Decryptor;
|
||||
import org.apache.poi.poifs.crypt.EncryptionInfo;
|
||||
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
|
||||
import org.apache.poi.xssf.eventusermodel.XSSFReader;
|
||||
import org.apache.poi.xssf.model.StylesTable;
|
||||
import org.apache.poi.ooxml.util.PackageHelper;
|
||||
import org.xml.sax.ContentHandler;
|
||||
import org.xml.sax.InputSource;
|
||||
import org.xml.sax.XMLReader;
|
||||
import org.xml.sax.helpers.XMLReaderFactory;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
@@ -36,6 +32,7 @@ import java.util.concurrent.FutureTask;
|
||||
|
||||
/**
|
||||
* created by wwang on 2012-9-27 Detailled comment
|
||||
*
|
||||
*/
|
||||
public class ExcelReader implements Callable {
|
||||
|
||||
@@ -49,8 +46,6 @@ public class ExcelReader implements Callable {
|
||||
|
||||
private String charset = "UTF-8";
|
||||
|
||||
private String password = null;
|
||||
|
||||
private java.io.InputStream is;
|
||||
|
||||
private List<String> sheetNames = new ArrayList<String>();
|
||||
@@ -69,17 +64,15 @@ public class ExcelReader implements Callable {
|
||||
task = new Thread(futureTask);
|
||||
}
|
||||
|
||||
public void parse(String fileURL, String charset, String password) {
|
||||
public void parse(String fileURL, String charset) {
|
||||
this.fileURL = fileURL;
|
||||
this.charset = charset;
|
||||
this.password = password;
|
||||
task.start();
|
||||
}
|
||||
|
||||
public void parse(java.io.InputStream is, String charset, String password) {
|
||||
public void parse(java.io.InputStream is, String charset) {
|
||||
this.is = is;
|
||||
this.charset = charset;
|
||||
this.password = password;
|
||||
task.start();
|
||||
}
|
||||
|
||||
@@ -127,25 +120,11 @@ public class ExcelReader implements Callable {
|
||||
|
||||
public Object call() throws Exception {
|
||||
OPCPackage pkg = null;
|
||||
POIFSFileSystem fs = null;
|
||||
try {
|
||||
if (password != null) {
|
||||
if (fileURL != null) {
|
||||
fs = new POIFSFileSystem(new File(fileURL));
|
||||
} else {
|
||||
fs = new POIFSFileSystem(is);
|
||||
}
|
||||
Decryptor d = Decryptor.getInstance(new EncryptionInfo(fs));
|
||||
if (!d.verifyPassword(password)) {
|
||||
throw new RuntimeException("Error: Cannot decrypt Excel file. Invalid password.");
|
||||
}
|
||||
pkg = OPCPackage.open(d.getDataStream(fs));
|
||||
if (fileURL != null) {
|
||||
pkg = OPCPackage.open(fileURL);
|
||||
} else {
|
||||
if (fileURL != null) {
|
||||
pkg = OPCPackage.open(fileURL);
|
||||
} else {
|
||||
pkg = PackageHelper.open(is);
|
||||
}
|
||||
pkg = PackageHelper.open(is);
|
||||
}
|
||||
XSSFReader r = new XSSFReader(pkg);
|
||||
|
||||
@@ -225,9 +204,6 @@ public class ExcelReader implements Callable {
|
||||
if (pkg != null) {
|
||||
pkg.revert();
|
||||
}
|
||||
if (fs != null) {
|
||||
fs.close();
|
||||
}
|
||||
cache.notifyErrorOccurred();
|
||||
}
|
||||
return null;
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend-soap</artifactId>
|
||||
<version>2.1-20190716</version>
|
||||
<version>2.1-20190513</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend-soap</name>
|
||||
|
||||
@@ -124,9 +124,7 @@ public class SOAPUtil {
|
||||
}
|
||||
SOAPMessage message = messageFactory.createMessage();
|
||||
MimeHeaders mimeHeaders = message.getMimeHeaders();
|
||||
|
||||
setSoapAction(version, soapAction, mimeHeaders);
|
||||
|
||||
mimeHeaders.setHeader("SOAPAction", soapAction);
|
||||
if (basicAuth) {
|
||||
addBasicAuthHeader(mimeHeaders, username, password);
|
||||
}
|
||||
@@ -265,8 +263,8 @@ public class SOAPUtil {
|
||||
}
|
||||
SOAPMessage message = messageFactory.createMessage();
|
||||
MimeHeaders mimeHeaders = message.getMimeHeaders();
|
||||
setSoapAction(version, soapAction, mimeHeaders);
|
||||
SOAPPart soapPart = message.getSOAPPart();
|
||||
mimeHeaders.setHeader("SOAPAction", soapAction);
|
||||
SOAPPart soapPart = message.getSOAPPart();
|
||||
|
||||
String encoding = getEncoding(soapMessage);
|
||||
|
||||
@@ -293,20 +291,8 @@ public class SOAPUtil {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/* https://jira.talendforge.org/browse/TDI-42581 skip add SOAPAction directly to header v1.2 */
|
||||
private void setSoapAction(String version, String soapAction, MimeHeaders mimeHeaders) {
|
||||
if (SOAP12.equals(version)) {
|
||||
// in soap version 1.2 param 'action' optional and should not be empty
|
||||
if( soapAction != null && !soapAction.trim().isEmpty()) {
|
||||
mimeHeaders.setHeader("Content-Type", "application/soap+xml; charset=utf-8; action=\"" + soapAction + "\"");
|
||||
}
|
||||
} else {
|
||||
mimeHeaders.setHeader("SOAPAction", soapAction);
|
||||
}
|
||||
}
|
||||
|
||||
private String getEncoding(String text) {
|
||||
|
||||
private String getEncoding(String text) {
|
||||
String result = Charset.defaultCharset().name();
|
||||
|
||||
if(text == null) {
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend-codegen-utils</artifactId>
|
||||
<!-- release for revert version of library -->
|
||||
<version>0.28.0</version>
|
||||
<version>0.25.4</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
|
||||
@@ -51,8 +51,6 @@ public final class TypeConverter {
|
||||
|
||||
public static final String LIST = "id_List";
|
||||
|
||||
public static final String OBJECT = "id_Object";
|
||||
|
||||
private TypeConverter() {
|
||||
// Class provides static utility methods and shouldn't be instantiated
|
||||
}
|
||||
@@ -169,8 +167,6 @@ public final class TypeConverter {
|
||||
return SHORT;
|
||||
case "java.util.Date":
|
||||
return DATE;
|
||||
case "java.lang.Object":
|
||||
return OBJECT;
|
||||
default:
|
||||
throw new UnsupportedOperationException("Unrecognized java class " + javaClass);
|
||||
}
|
||||
|
||||
@@ -5,9 +5,9 @@
|
||||
>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend-db-exasol</artifactId>
|
||||
<version>2.1.4</version>
|
||||
<version>2.1.2</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend-db-exasol</name>
|
||||
|
||||
@@ -94,11 +94,12 @@ public class EXABulkUtil {
|
||||
private String createNumberFormat(Integer length, Integer precision, boolean hasGroups) {
|
||||
if (length != null && length.intValue() > 0) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = length - 1; i >= 0; i--) {
|
||||
if(hasGroups && i < length - 1 && i > 0 && (i % 3 == 2)) {
|
||||
sb.append("G");
|
||||
}
|
||||
sb.append("9");
|
||||
int numGroups = (length.intValue() / 3) + 1;
|
||||
for (int i = 0; i < numGroups; i++) {
|
||||
if (i > 0 && hasGroups) {
|
||||
sb.append("G");
|
||||
}
|
||||
sb.append("999");
|
||||
}
|
||||
if (precision != null && precision.intValue() > 0) {
|
||||
sb.append("D");
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talendExcel-1.5-20190731</artifactId>
|
||||
<artifactId>talendExcel-1.4-20190531</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ package org.talend;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.Date;
|
||||
@@ -11,23 +10,17 @@ import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.poi.poifs.crypt.EncryptionInfo;
|
||||
import org.apache.poi.poifs.crypt.EncryptionMode;
|
||||
import org.apache.poi.poifs.crypt.Encryptor;
|
||||
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.CellStyle;
|
||||
import org.apache.poi.ss.usermodel.CellType;
|
||||
import org.apache.poi.ss.usermodel.Font;
|
||||
import org.apache.poi.ss.usermodel.FormulaEvaluator;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.apache.poi.xssf.streaming.SXSSFSheet;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbookType;
|
||||
import org.apache.poi.ss.usermodel.FormulaEvaluator;
|
||||
|
||||
public class ExcelTool {
|
||||
|
||||
@@ -77,8 +70,6 @@ public class ExcelTool {
|
||||
|
||||
private boolean isTrackAllColumns = false;
|
||||
|
||||
private String password = null;
|
||||
|
||||
public ExcelTool() {
|
||||
cellStylesMapping = new HashMap<>();
|
||||
}
|
||||
@@ -119,7 +110,21 @@ public class ExcelTool {
|
||||
initPreXlsx(fileName);
|
||||
}
|
||||
if (appendWorkbook) {
|
||||
appendActionForFile(fileName);
|
||||
InputStream inp = new FileInputStream(fileName);
|
||||
wb = WorkbookFactory.create(inp);
|
||||
sheet = wb.getSheet(sheetName);
|
||||
if (sheet != null) {
|
||||
if (appendSheet) {
|
||||
if (sheet.getLastRowNum() != 0 || sheet.getRow(0) != null) {
|
||||
curY = sheet.getLastRowNum() + 1;
|
||||
}
|
||||
} else {
|
||||
wb.removeSheetAt(wb.getSheetIndex(sheetName));
|
||||
sheet = wb.createSheet(sheetName);
|
||||
}
|
||||
} else {
|
||||
sheet = wb.createSheet(sheetName);
|
||||
}
|
||||
} else {
|
||||
xlsxFile.delete();
|
||||
wb = new SXSSFWorkbook(rowAccessWindowSize);
|
||||
@@ -135,47 +140,6 @@ public class ExcelTool {
|
||||
}
|
||||
}
|
||||
|
||||
public void prepareXlsmFile(String fileName) throws Exception {
|
||||
File xlsmFile = new File(fileName);
|
||||
if (xlsmFile.exists()) {
|
||||
if (isAbsY && keepCellFormat) {
|
||||
initPreXlsx(fileName);
|
||||
}
|
||||
if (appendWorkbook) {
|
||||
appendActionForFile(fileName);
|
||||
} else {
|
||||
xlsmFile.delete();
|
||||
wb = new SXSSFWorkbook(new XSSFWorkbook(XSSFWorkbookType.XLSM), rowAccessWindowSize);
|
||||
sheet = wb.createSheet(sheetName);
|
||||
}
|
||||
} else {
|
||||
wb = new SXSSFWorkbook(new XSSFWorkbook(XSSFWorkbookType.XLSM), rowAccessWindowSize);
|
||||
sheet = wb.createSheet(sheetName);
|
||||
}
|
||||
if (isAbsY) {
|
||||
startX = absX;
|
||||
curY = absY;
|
||||
}
|
||||
}
|
||||
|
||||
private void appendActionForFile(String fileName) throws IOException {
|
||||
InputStream inp = new FileInputStream(fileName);
|
||||
wb = WorkbookFactory.create(inp);
|
||||
sheet = wb.getSheet(sheetName);
|
||||
if (sheet != null) {
|
||||
if (appendSheet) {
|
||||
if (sheet.getLastRowNum() != 0 || sheet.getRow(0) != null) {
|
||||
curY = sheet.getLastRowNum() + 1;
|
||||
}
|
||||
} else {
|
||||
wb.removeSheetAt(wb.getSheetIndex(sheetName));
|
||||
sheet = wb.createSheet(sheetName);
|
||||
}
|
||||
} else {
|
||||
sheet = wb.createSheet(sheetName);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @return start insert row index.
|
||||
@@ -338,14 +302,13 @@ public class ExcelTool {
|
||||
}
|
||||
|
||||
public void writeExcel(OutputStream outputStream) throws Exception {
|
||||
try {
|
||||
wb.write(outputStream);
|
||||
wb.close();
|
||||
} finally {
|
||||
if (outputStream != null) {
|
||||
outputStream.close();
|
||||
}
|
||||
}
|
||||
try {
|
||||
wb.write(outputStream);
|
||||
} finally {
|
||||
if(outputStream != null) {
|
||||
outputStream.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void writeExcel(String fileName, boolean createDir) throws Exception {
|
||||
@@ -356,29 +319,14 @@ public class ExcelTool {
|
||||
pFile.mkdirs();
|
||||
}
|
||||
}
|
||||
FileOutputStream fileOutput = new FileOutputStream(fileName);
|
||||
if (appendWorkbook && appendSheet && recalculateFormula) {
|
||||
evaluateFormulaCell();
|
||||
}
|
||||
FileOutputStream fileOutput = new FileOutputStream(fileName);
|
||||
POIFSFileSystem fs = null;
|
||||
try {
|
||||
if (password == null) {
|
||||
wb.write(fileOutput);
|
||||
} else {
|
||||
fs = new POIFSFileSystem();
|
||||
Encryptor encryptor = new EncryptionInfo(EncryptionMode.agile).getEncryptor();
|
||||
encryptor.confirmPassword(password);
|
||||
OutputStream encryptedDataStream = encryptor.getDataStream(fs);
|
||||
wb.write(encryptedDataStream);
|
||||
encryptedDataStream.close(); // this is mandatory to do that at that point
|
||||
fs.writeFilesystem(fileOutput);
|
||||
}
|
||||
wb.write(fileOutput);
|
||||
} finally {
|
||||
wb.close();
|
||||
fileOutput.close();
|
||||
if (fs != null) {
|
||||
fs.close();
|
||||
}
|
||||
fileOutput.close();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -401,9 +349,4 @@ public class ExcelTool {
|
||||
((SXSSFSheet) sheet).flushRows();
|
||||
}
|
||||
}
|
||||
|
||||
public void setPasswordProtection(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talendzip</artifactId>
|
||||
<version>1.0-20190917</version>
|
||||
<version>1.0-20190527</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
@@ -40,18 +40,18 @@
|
||||
<dependency>
|
||||
<groupId>net.lingala.zip4j</groupId>
|
||||
<artifactId>zip4j</artifactId>
|
||||
<version>1.3.3</version>
|
||||
<version>1.3.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<version>1.19</version>
|
||||
<version>1.10</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>checkArchive-1.1-20190917</artifactId>
|
||||
<artifactId>checkArchive-1.1-20181130</artifactId>
|
||||
<version>6.0.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
package com.talend.compress.zip;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class Util {
|
||||
public java.util.List<UnzippedFile> unzippedFiles = new java.util.ArrayList<UnzippedFile>();
|
||||
|
||||
@@ -55,10 +53,6 @@ public class Util {
|
||||
f = new java.io.File(path, shortName);
|
||||
}
|
||||
|
||||
if(!f.getCanonicalPath().startsWith(new java.io.File(path).getCanonicalPath())) {
|
||||
throw new IOException("expanding " + f.getName()
|
||||
+ " would create file outside of " + path);
|
||||
}
|
||||
checkDir(f);
|
||||
|
||||
f.getParentFile().mkdirs();
|
||||
@@ -95,10 +89,6 @@ public class Util {
|
||||
f = new java.io.File(path, shortName);
|
||||
}
|
||||
|
||||
if(!f.getCanonicalPath().startsWith(new java.io.File(path).getCanonicalPath())) {
|
||||
throw new IOException("expanding " + f.getName()
|
||||
+ " would create file outside of " + path);
|
||||
}
|
||||
if (isDirectory) {
|
||||
if (!f.exists()) {
|
||||
f.mkdirs();
|
||||
|
||||
@@ -5,9 +5,9 @@
|
||||
>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>thashfile</artifactId>
|
||||
<version>3.1-20190910</version>
|
||||
<version>3.0-20170711</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>thashfile</name>
|
||||
@@ -45,8 +45,8 @@
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.5.1</version>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
<source>1.7</source>
|
||||
<target>1.7</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
|
||||
@@ -9,44 +9,45 @@ import java.util.concurrent.ConcurrentHashMap;
|
||||
import org.talend.designer.components.hashfile.memory.AdvancedMemoryHashFile;
|
||||
|
||||
public class MapHashFile {
|
||||
// use this map instead of globalMap
|
||||
//use this map instead of globalMap
|
||||
private Map<String, AdvancedMemoryHashFile> resourceMap = new ConcurrentHashMap<>();
|
||||
// keep the present key of AdvancedMemoryHashFile as key and the previous key as
|
||||
// value
|
||||
//keep the present key of AdvancedMemoryHashFile as key and the previous key as value
|
||||
private Map<String, String> keyMap = new ConcurrentHashMap<>();
|
||||
// singleton
|
||||
//singleton
|
||||
private static final MapHashFile mhf = new MapHashFile();
|
||||
|
||||
public static TalendMultiThreadLockMap resourceLockMap = new TalendMultiThreadLockMap();
|
||||
public static TalendMultiThreadLockMap resourceLockMap = new TalendMultiThreadLockMap();
|
||||
|
||||
public static class TalendMultiThreadLockMap {
|
||||
public static class TalendMultiThreadLockMap {
|
||||
|
||||
private Map<Object, Object> tMultiTheadLockMap = new ConcurrentHashMap<>();
|
||||
|
||||
public Object get(Object key) {
|
||||
return tMultiTheadLockMap.computeIfAbsent(key, k -> new Object());
|
||||
}
|
||||
|
||||
public void remove(Object key) {
|
||||
tMultiTheadLockMap.remove(key);
|
||||
}
|
||||
}
|
||||
private Map<Object, Object> tMultiTheadLockMap = new HashMap<Object, Object>();
|
||||
|
||||
public synchronized Object get(Object key) {
|
||||
if (tMultiTheadLockMap.get(key) == null) {
|
||||
tMultiTheadLockMap.put(key, new Object());
|
||||
}
|
||||
return tMultiTheadLockMap.get(key);
|
||||
}
|
||||
|
||||
public synchronized void remove(Object key){
|
||||
tMultiTheadLockMap.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
private MapHashFile() {
|
||||
}
|
||||
|
||||
public static MapHashFile getMapHashFile() {
|
||||
return mhf;
|
||||
}
|
||||
|
||||
// get the linked AdvancedMemoryHashFile
|
||||
|
||||
//get the linked AdvancedMemoryHashFile
|
||||
public AdvancedMemoryHashFile getAdvancedMemoryHashFile(String key) {
|
||||
AdvancedMemoryHashFile amhf = resourceMap.get(key);
|
||||
String prekey = keyMap.get(key);
|
||||
// if present AdvancedMemoryHashFile is null get the AdvancedMemoryHashFile
|
||||
// before present.
|
||||
//if present AdvancedMemoryHashFile is null get the AdvancedMemoryHashFile before present.
|
||||
int size = keyMap.size();
|
||||
while (amhf == null && (size--) > 0) {
|
||||
while(amhf==null && (size--)>0){
|
||||
amhf = resourceMap.get(prekey);
|
||||
prekey = keyMap.get(prekey);
|
||||
}
|
||||
@@ -60,18 +61,16 @@ public class MapHashFile {
|
||||
public Map<String, String> getKeyMap() {
|
||||
return keyMap;
|
||||
}
|
||||
|
||||
public void clearCache(String key) {
|
||||
public void clearCache(String key){
|
||||
clearChildCache(getRootCache(key));
|
||||
}
|
||||
|
||||
public void clearChildCache(String root) {
|
||||
public void clearChildCache(String root){
|
||||
Set<String> set = keyMap.keySet();
|
||||
synchronized (keyMap) {
|
||||
synchronized(keyMap) {
|
||||
Iterator<String> it = set.iterator();
|
||||
while (it.hasNext()) {
|
||||
while(it.hasNext()){
|
||||
String key = it.next();
|
||||
if (root.equals(keyMap.get(key))) {
|
||||
if(root.equals(keyMap.get(key))){
|
||||
this.resourceMap.remove(key);
|
||||
this.keyMap.remove(key);
|
||||
clearChildCache(key);
|
||||
@@ -80,11 +79,11 @@ public class MapHashFile {
|
||||
}
|
||||
this.resourceMap.remove(root);
|
||||
}
|
||||
|
||||
public String getRootCache(String cache) {
|
||||
|
||||
public String getRootCache(String cache){
|
||||
String root;
|
||||
while ((root = keyMap.get(cache)) != null) {
|
||||
cache = root;
|
||||
while((root = keyMap.get(cache))!=null){
|
||||
cache=root;
|
||||
}
|
||||
return cache;
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-M2</version>
|
||||
<version>7.3.1-SNAPSHOT</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.components.libs</artifactId>
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
|
||||
<FAMILIES>
|
||||
<FAMILY>Databases/DB Specifics/Amazon/Redshift</FAMILY>
|
||||
<FAMILY>Cloud/Amazon/Redshift</FAMILY>
|
||||
</FAMILIES>
|
||||
|
||||
<DOCUMENTATION>
|
||||
|
||||
@@ -24,15 +24,13 @@
|
||||
String dataset = ElementParameterParser.getValue(node, "__DATASET__");
|
||||
String table = ElementParameterParser.getValue(node, "__TABLE__");
|
||||
String gsFile = ElementParameterParser.getValue(node, "__GS_FILE__");
|
||||
boolean setFieldDelimiter = ElementParameterParser.getBooleanValue(node, "__SET_FIELD_DELIMITER__");
|
||||
String fieldDelimiter = ElementParameterParser.getValue(node, "__FIELD_DELIMITER__");
|
||||
String actionOnData = ElementParameterParser.getValue(node, "__ACTION_ON_DATA__");
|
||||
boolean dieOnError = ElementParameterParser.getBooleanValue(node, "__DIE_ON_ERROR__");
|
||||
boolean createTableIfNotExist = ElementParameterParser.getBooleanValue(node, "__CREATE_TABLE_IF_NOT_EXIST__");
|
||||
String gsFileHeader = ElementParameterParser.getValue(node, "__GS_FILE_HEADER__");
|
||||
boolean dieOnError = "true".equals(ElementParameterParser.getValue(node, "__DIE_ON_ERROR__"));
|
||||
|
||||
String tokenFile = ElementParameterParser.getValue(node,"__TOKEN_NAME__");
|
||||
boolean isLog4jEnabled = ElementParameterParser.getBooleanValue(node.getProcess(), "__LOG4J_ACTIVATE__");
|
||||
boolean dropTable = ElementParameterParser.getBooleanValue(node, "__DROP__");
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
|
||||
String passwordFieldName = "";
|
||||
|
||||
@@ -203,7 +201,7 @@
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
}
|
||||
boolean bulkFileAlreadyExists = ElementParameterParser.getBooleanValue(node, "__BULK_FILE_ALREADY_EXIST__");
|
||||
boolean bulkFileAlreadyExists = "true".equals(ElementParameterParser.getValue(node, "__BULK_FILE_ALREADY_EXIST__"));
|
||||
String accessKey = ElementParameterParser.getValue(node, "__GS_ACCESS_KEY__");
|
||||
String secretKey = ElementParameterParser.getValue(node, "__GS_SECRET_KEY__");
|
||||
String localFilename = ElementParameterParser.getValue(node, "__GS_LOCAL_FILE__");
|
||||
@@ -236,7 +234,6 @@
|
||||
}
|
||||
%>
|
||||
gsService_<%=cid%>.putObject(<%=bucketName%>, fileObject_<%=cid%>);
|
||||
/* ----END-UPLOADING-FILE---- */
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
@@ -245,6 +242,7 @@
|
||||
}
|
||||
}
|
||||
%>
|
||||
/* ----END-UPLOADING-FILE---- */
|
||||
|
||||
<%
|
||||
if (authMode.equals("OAUTH")) {
|
||||
@@ -260,131 +258,95 @@
|
||||
|
||||
com.google.api.services.bigquery.model.JobConfiguration config_<%=cid%> = new com.google.api.services.bigquery.model.JobConfiguration();
|
||||
com.google.api.services.bigquery.model.JobConfigurationLoad queryLoad_<%=cid%> = new com.google.api.services.bigquery.model.JobConfigurationLoad();
|
||||
|
||||
<%if (dropTable) {%>
|
||||
try {
|
||||
bigqueryclient_<%=cid%>.tables().delete(PROJECT_ID_<%=cid%>, <%=dataset%>, <%=table%>).execute();
|
||||
} catch (com.google.api.client.googleapis.json.GoogleJsonResponseException e_<%=cid%>) {
|
||||
if (e_<%=cid%>.getDetails().getCode() != 404) {
|
||||
<%
|
||||
if(dieOnError) {
|
||||
%>
|
||||
throw e_<%=cid%>;
|
||||
<%
|
||||
} else {
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.error("<%=cid%> - "+ e_<%=cid%>.getDetails().getMessage());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
System.err.println(e_<%=cid%>.getDetails().getMessage());
|
||||
<%
|
||||
com.google.api.services.bigquery.model.TableSchema schema_<%=cid%> = new com.google.api.services.bigquery.model.TableSchema();
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Table field schema:");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
java.util.List<com.google.api.services.bigquery.model.TableFieldSchema> fields_<%=cid%> = new java.util.ArrayList<com.google.api.services.bigquery.model.TableFieldSchema>();
|
||||
<%
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
if ((metadatas!=null) && (metadatas.size() > 0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
if (metadata != null) {
|
||||
List<IMetadataColumn> columns = metadata.getListColumns();
|
||||
int nbColumns = columns.size();
|
||||
for (int i = 0; i < nbColumns; i++ ) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
String columnName = column.getLabel();
|
||||
String typeToGenerate = "string";
|
||||
if("id_Float".equals(column.getTalendType()) || "id_Double".equals(column.getTalendType())) {
|
||||
typeToGenerate = "float";
|
||||
}else if("id_Integer".equals(column.getTalendType()) || "id_Long".equals(column.getTalendType()) || "id_Short".equals(column.getTalendType())) {
|
||||
typeToGenerate = "integer";
|
||||
} else if("id_Character".equals(column.getTalendType())) {
|
||||
typeToGenerate = "string";
|
||||
} else if("id_BigDecimal".equals(column.getTalendType())) {
|
||||
typeToGenerate = "numeric";
|
||||
} else if("id_Boolean".equals(column.getTalendType())) {
|
||||
typeToGenerate = "boolean";
|
||||
} else if("id_Date".equals(column.getTalendType())) {
|
||||
String pattern = column.getPattern();
|
||||
if(pattern.length() == 12 || pattern.isEmpty() || "\"\"".equals(pattern)) {
|
||||
typeToGenerate = "date";
|
||||
}else if(pattern.length() > 12){
|
||||
typeToGenerate = "timestamp";
|
||||
}else{
|
||||
typeToGenerate = "string";
|
||||
}
|
||||
}
|
||||
%>
|
||||
}
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if (createTableIfNotExist) { %>
|
||||
|
||||
try {
|
||||
com.google.api.services.bigquery.model.Table getTable = bigqueryclient_<%=cid%>.tables().get(PROJECT_ID_<%=cid%>, <%=dataset%>, <%=table%>).execute();
|
||||
queryLoad_<%=cid%>.setCreateDisposition("CREATE_NEVER");
|
||||
} catch (com.google.api.client.googleapis.json.GoogleJsonResponseException e_<%=cid%>){
|
||||
if (e_<%=cid%>.getDetails().getCode() == 404) {
|
||||
com.google.api.services.bigquery.model.TableSchema schema_<%=cid%> = new com.google.api.services.bigquery.model.TableSchema();
|
||||
<%
|
||||
String modeType = null;
|
||||
if (!column.isNullable()) {
|
||||
modeType = "REQUIRED";
|
||||
} else {
|
||||
modeType = "NULLABLE";
|
||||
}
|
||||
%>
|
||||
com.google.api.services.bigquery.model.TableFieldSchema <%=columnName%>_<%=cid%> = new com.google.api.services.bigquery.model.TableFieldSchema();
|
||||
<%=columnName%>_<%=cid%>.setName("<%=columnName%>");
|
||||
<%=columnName%>_<%=cid%>.setType("<%=typeToGenerate%>");
|
||||
<%=columnName%>_<%=cid%>.setMode("<%=modeType%>");
|
||||
fields_<%=cid%>.add(<%=columnName%>_<%=cid%>);
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Table field schema:");
|
||||
log.debug("<%=cid%> - Field index[<%=i%>] {\"name\":\"<%=columnName%>\",\"type\":\"<%=typeToGenerate%>\",\"mode\":\"<%=modeType%>\"}");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
java.util.List<com.google.api.services.bigquery.model.TableFieldSchema> fields_<%=cid%> = new java.util.ArrayList<com.google.api.services.bigquery.model.TableFieldSchema>();
|
||||
<%
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
if ((metadatas!=null) && (metadatas.size() > 0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
if (metadata != null) {
|
||||
List<IMetadataColumn> columns = metadata.getListColumns();
|
||||
int nbColumns = columns.size();
|
||||
for (int i = 0; i < nbColumns; i++ ) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
String columnName = column.getLabel();
|
||||
String typeToGenerate = "string";
|
||||
if("id_Float".equals(column.getTalendType()) || "id_Double".equals(column.getTalendType())) {
|
||||
typeToGenerate = "float";
|
||||
}else if("id_Integer".equals(column.getTalendType()) || "id_Long".equals(column.getTalendType()) || "id_Short".equals(column.getTalendType())) {
|
||||
typeToGenerate = "integer";
|
||||
} else if("id_Character".equals(column.getTalendType())) {
|
||||
typeToGenerate = "string";
|
||||
} else if("id_BigDecimal".equals(column.getTalendType())) {
|
||||
typeToGenerate = "numeric";
|
||||
} else if("id_Boolean".equals(column.getTalendType())) {
|
||||
typeToGenerate = "boolean";
|
||||
} else if("id_Date".equals(column.getTalendType())) {
|
||||
String pattern = column.getPattern();
|
||||
if(pattern.length() == 12 || pattern.isEmpty() || "\"\"".equals(pattern)) {
|
||||
typeToGenerate = "date";
|
||||
}else if(pattern.length() > 12){
|
||||
typeToGenerate = "timestamp";
|
||||
}else{
|
||||
typeToGenerate = "string";
|
||||
}
|
||||
}
|
||||
String modeType = (!column.isNullable()) ? "REQUIRED" : "NULLABLE";
|
||||
%>
|
||||
com.google.api.services.bigquery.model.TableFieldSchema <%=columnName%>_<%=cid%> = new com.google.api.services.bigquery.model.TableFieldSchema();
|
||||
<%=columnName%>_<%=cid%>.setName("<%=columnName%>");
|
||||
<%=columnName%>_<%=cid%>.setType("<%=typeToGenerate%>");
|
||||
<%=columnName%>_<%=cid%>.setMode("<%=modeType%>");
|
||||
fields_<%=cid%>.add(<%=columnName%>_<%=cid%>);
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.debug("<%=cid%> - Field index[<%=i%>] {\"name\":\"<%=columnName%>\",\"type\":\"<%=typeToGenerate%>\",\"mode\":\"<%=modeType%>\"}");
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
schema_<%=cid%>.setFields(fields_<%=cid%>);
|
||||
queryLoad_<%=cid%>.setSchema(schema_<%=cid%>);
|
||||
queryLoad_<%=cid%>.setCreateDisposition("CREATE_IF_NEEDED");
|
||||
} else {
|
||||
<%
|
||||
if(dieOnError) {
|
||||
%>
|
||||
throw e_<%=cid%>;
|
||||
<%
|
||||
} else {
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.error("<%=cid%> - "+ e_<%=cid%>.getDetails().getMessage());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
System.err.println(e_<%=cid%>.getDetails().getMessage());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
|
||||
schema_<%=cid%>.setFields(fields_<%=cid%>);
|
||||
|
||||
queryLoad_<%=cid%>.setSchema(schema_<%=cid%>);
|
||||
<%
|
||||
if("true".equals(ElementParameterParser.getValue(node, "__CREATE_TABLE_IF_NOT_EXIST__"))) {
|
||||
%>
|
||||
queryLoad_<%=cid%>.setCreateDisposition("CREATE_IF_NEEDED");
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
queryLoad_<%=cid%>.setCreateDisposition("CREATE_NEVER");
|
||||
<%
|
||||
}
|
||||
if(setFieldDelimiter) {
|
||||
|
||||
if("true".equals(ElementParameterParser.getValue(node, "__SET_FIELD_DELIMITER__"))) {
|
||||
%>
|
||||
queryLoad_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
queryLoad_<%=cid%>.setAllowQuotedNewlines(true);
|
||||
|
||||
queryLoad_<%=cid%>.setWriteDisposition("WRITE_<%=actionOnData%>");
|
||||
com.google.api.services.bigquery.model.TableReference destinationTable_<%=cid%> = new com.google.api.services.bigquery.model.TableReference();
|
||||
destinationTable_<%=cid%>.setProjectId(PROJECT_ID_<%=cid%>);
|
||||
@@ -392,8 +354,8 @@
|
||||
destinationTable_<%=cid%>.setTableId(<%=table%>);
|
||||
|
||||
queryLoad_<%=cid%>.setDestinationTable(destinationTable_<%=cid%>);
|
||||
queryLoad_<%=cid%>.setSourceUris(java.util.Arrays.asList(<%=gsFile%>));
|
||||
queryLoad_<%=cid%>.setSkipLeadingRows(<%=gsFileHeader%>);
|
||||
queryLoad_<%=cid%>.setSourceUris(java.util.Arrays.asList(<%=ElementParameterParser.getValue(node, "__GS_FILE__")%>));
|
||||
queryLoad_<%=cid%>.setSkipLeadingRows(<%=ElementParameterParser.getValue(node, "__GS_FILE_HEADER__")%>);
|
||||
queryLoad_<%=cid%>.setNullMarker("\\N");
|
||||
config_<%=cid%>.setLoad(queryLoad_<%=cid%>);
|
||||
|
||||
@@ -497,11 +459,9 @@
|
||||
%>
|
||||
/* ----START-CREATING-JOB (Cloud API)---- */
|
||||
com.google.cloud.bigquery.TableId tableId_<%=cid%> = com.google.cloud.bigquery.TableId.of(<%=projectId%>, <%=dataset%>, <%=table%>);
|
||||
com.google.cloud.bigquery.Table table_<%=cid%> = bigquery_<%=cid%>.getTable(tableId_<%=cid%>);
|
||||
com.google.cloud.bigquery.LoadJobConfiguration.Builder loadJobBuilder_<%=cid%> = com.google.cloud.bigquery.LoadJobConfiguration.newBuilder(tableId_<%=cid%>, <%=gsFile%>);
|
||||
|
||||
boolean dropTable_<%=cid%> = <%=dropTable%>;
|
||||
|
||||
if (dropTable_<%=cid%> && bigquery_<%=cid%>.getTable(tableId_<%=cid%>) != null) {
|
||||
if (<%=ElementParameterParser.getBooleanValue(node, "__DROP__")%> && table_<%=cid%> != null) {
|
||||
boolean deleted = bigquery_<%=cid%>.delete(tableId_<%=cid%>);
|
||||
if (deleted) {
|
||||
<%
|
||||
@@ -515,100 +475,123 @@
|
||||
throw new RuntimeException("Unable to delete table " + tableId_<%=cid%>);
|
||||
}
|
||||
}
|
||||
|
||||
<%if(createTableIfNotExist){%>
|
||||
if(bigquery_<%=cid%>.getTable(tableId_<%=cid%>) == null){
|
||||
java.util.List<com.google.cloud.bigquery.Field> fields_<%=cid%> = new java.util.ArrayList<>();
|
||||
<%
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
if ((metadatas!=null) && (metadatas.size() > 0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
if (metadata != null) {
|
||||
List<IMetadataColumn> columns = metadata.getListColumns();
|
||||
int nbColumns = columns.size();
|
||||
for (int i = 0; i < nbColumns; i++ ) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
String columnName = column.getLabel();
|
||||
String typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
|
||||
if("id_String".equals(column.getTalendType()) || "id_Character".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
|
||||
} else if ("id_Float".equals(column.getTalendType()) || "id_Double".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.FLOAT";
|
||||
} else if ("id_Short".equals(column.getTalendType()) || "id_Integer".equals(column.getTalendType()) || "id_Long".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.INTEGER";
|
||||
} else if ("id_BigDecimal".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.NUMERIC";
|
||||
} else if ("id_Boolean".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.BOOLEAN";
|
||||
} else if ("id_Date".equals(column.getTalendType())) {
|
||||
String pattern = column.getPattern();
|
||||
if(pattern.length() == 12 || pattern.isEmpty() || "\"\"".equals(pattern)) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.DATE";
|
||||
}else if(pattern.length() > 12){
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.TIMESTAMP";
|
||||
}else{
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
|
||||
}
|
||||
}
|
||||
String modeType = (!column.isNullable()) ? "REQUIRED" : "NULLABLE";
|
||||
if (<%=ElementParameterParser.getBooleanValue(node, "__DROP__")%> || <%=ElementParameterParser.getBooleanValue(node, "__CREATE_TABLE_IF_NOT_EXIST__")%>) {
|
||||
java.util.List<com.google.cloud.bigquery.Field> fields_<%=cid%> = new java.util.ArrayList<>();
|
||||
<%
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
if ((metadatas!=null) && (metadatas.size() > 0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
if (metadata != null) {
|
||||
List<IMetadataColumn> columns = metadata.getListColumns();
|
||||
int nbColumns = columns.size();
|
||||
for (int i = 0; i < nbColumns; i++ ) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
String columnName = column.getLabel();
|
||||
String typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
|
||||
if("id_String".equals(column.getTalendType()) || "id_Character".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
|
||||
} else if ("id_Float".equals(column.getTalendType()) || "id_Double".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.FLOAT";
|
||||
} else if ("id_Short".equals(column.getTalendType()) || "id_Integer".equals(column.getTalendType()) || "id_Long".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.INTEGER";
|
||||
} else if ("id_BigDecimal".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.NUMERIC";
|
||||
} else if ("id_Boolean".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.BOOLEAN";
|
||||
} else if ("id_Date".equals(column.getTalendType())) {
|
||||
String pattern = column.getPattern();
|
||||
if(pattern.length() == 12 || pattern.isEmpty() || "\"\"".equals(pattern)) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.DATE";
|
||||
}else if(pattern.length() > 12){
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.TIMESTAMP";
|
||||
}else{
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
|
||||
}
|
||||
}
|
||||
|
||||
String modeType = "NULLABLE";
|
||||
if (!column.isNullable()) {
|
||||
modeType = "REQUIRED";
|
||||
}
|
||||
%>
|
||||
|
||||
com.google.cloud.bigquery.Field field_<%=i%> = com.google.cloud.bigquery.Field.newBuilder("<%=columnName%>", <%=typeToGenerate%>)
|
||||
.setMode(com.google.cloud.bigquery.Field.Mode.valueOf("<%=modeType%>"))
|
||||
.build();
|
||||
fields_<%=cid%>.add(field_<%=i%>);
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.debug("<%=cid%> - Field index[<%=i%>] {\"name\":\"<%=columnName%>\",\"type\":\"<%=typeToGenerate%>\"}");
|
||||
<%
|
||||
}
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.debug("<%=cid%> - Field index[<%=i%>] {\"name\":\"<%=columnName%>\",\"type\":\"<%=typeToGenerate%>\"}");
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
com.google.cloud.bigquery.Schema schema_<%=cid%> = com.google.cloud.bigquery.Schema.of(fields_<%=cid%>);
|
||||
com.google.cloud.bigquery.TableInfo tableInfo_<%=cid%> = com.google.cloud.bigquery.TableInfo.newBuilder(tableId_<%=cid%>, com.google.cloud.bigquery.StandardTableDefinition.of(schema_<%=cid%>)).build();
|
||||
com.google.cloud.bigquery.Table table_<%=cid%> = bigquery_<%=cid%>.create(tableInfo_<%=cid%>);
|
||||
loadJobBuilder_<%=cid%>.setSchema(schema_<%=cid%>);
|
||||
loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_IF_NEEDED);
|
||||
|
||||
} else {
|
||||
loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_NEVER);
|
||||
}
|
||||
<%} else {%>
|
||||
loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_NEVER);
|
||||
<%}%>
|
||||
%>
|
||||
|
||||
loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_<%=actionOnData%>);
|
||||
loadJobBuilder_<%=cid%>.setDestinationTable(tableId_<%=cid%>);
|
||||
com.google.cloud.bigquery.CsvOptions.Builder csvOptions_<%=cid%> = com.google.cloud.bigquery.CsvOptions.newBuilder();
|
||||
csvOptions_<%=cid%>.setAllowQuotedNewLines(true);
|
||||
csvOptions_<%=cid%>.setSkipLeadingRows(<%=gsFileHeader%>);
|
||||
|
||||
<%if(setFieldDelimiter) {
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setFormatOptions(csvOptions_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>).build());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
com.google.cloud.bigquery.Schema schema_<%=cid%> = com.google.cloud.bigquery.Schema.of(fields_<%=cid%>);
|
||||
com.google.cloud.bigquery.TableInfo tableInfo_<%=cid%> = com.google.cloud.bigquery.TableInfo.newBuilder(tableId_<%=cid%>, com.google.cloud.bigquery.StandardTableDefinition.of(schema_<%=cid%>)).build();
|
||||
table_<%=cid%> = bigquery_<%=cid%>.create(tableInfo_<%=cid%>);
|
||||
loadJobBuilder_<%=cid%>.setSchema(schema_<%=cid%>);
|
||||
}
|
||||
|
||||
loadJobBuilder_<%=cid%>.setNullMarker("\\N");
|
||||
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.of(loadJobBuilder_<%=cid%>.build()));
|
||||
job_<%=cid%> = job_<%=cid%>.waitFor(com.google.cloud.RetryOption.initialRetryDelay(org.threeten.bp.Duration.ofSeconds(1)));
|
||||
if (job_<%=cid%> != null && job_<%=cid%>.getStatus().getError() == null) {
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Job execution status: " + job_<%=cid%>.getStatus());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%
|
||||
if("true".equals(ElementParameterParser.getValue(node, "__CREATE_TABLE_IF_NOT_EXIST__"))) {
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_IF_NEEDED);
|
||||
<%
|
||||
} else {
|
||||
List<com.google.cloud.bigquery.BigQueryError> errorList = job_<%=cid%>.getStatus().getExecutionErrors();
|
||||
throw new RuntimeException("Job failed: " + errorList.get(errorList.size() - 1));
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_NEVER);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
<%
|
||||
if("APPEND".equals(actionOnData)) {
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_APPEND);
|
||||
<%
|
||||
} else if("TRUNCATE".equals(actionOnData)) {
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_TRUNCATE);
|
||||
|
||||
<%
|
||||
}
|
||||
else {
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_EMPTY);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
loadJobBuilder_<%=cid%>.setDestinationTable(tableId_<%=cid%>);
|
||||
com.google.cloud.bigquery.CsvOptions.Builder csvOptions_<%=cid%> = com.google.cloud.bigquery.CsvOptions.newBuilder();
|
||||
csvOptions_<%=cid%>.setAllowQuotedNewLines(true);
|
||||
|
||||
<%if("true".equals(ElementParameterParser.getValue(node, "__SET_FIELD_DELIMITER__"))) {
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setFormatOptions(csvOptions_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>).build());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
loadJobBuilder_<%=cid%>.setNullMarker("\\N");
|
||||
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.of(loadJobBuilder_<%=cid%>.build()));
|
||||
job_<%=cid%> = job_<%=cid%>.waitFor(com.google.cloud.RetryOption.initialRetryDelay(org.threeten.bp.Duration.ofSeconds(1)));
|
||||
if (job_<%=cid%> != null && job_<%=cid%>.getStatus().getError() == null) {
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Job execution status: " + job_<%=cid%>.getStatus());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} else {
|
||||
List<com.google.cloud.bigquery.BigQueryError> errorList = job_<%=cid%>.getStatus().getExecutionErrors();
|
||||
throw new RuntimeException("Job failed: " + errorList.get(errorList.size() - 1));
|
||||
}
|
||||
|
||||
/* ----END-CREATING-JOB (Cloud API)---- */
|
||||
<%
|
||||
|
||||
@@ -173,7 +173,7 @@
|
||||
SHOW_IF="(SET_FIELD_DELIMITER == 'true')">
|
||||
<DEFAULT>","</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="25">
|
||||
<PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="25" SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'">
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="ENCODING" FIELD="ENCODING_TYPE" NUM_ROW="30"
|
||||
|
||||
@@ -221,9 +221,10 @@
|
||||
}
|
||||
|
||||
String query_<%=cid%> = <%=query%>;
|
||||
<% if(isLog4jEnabled) { %>
|
||||
log.info("<%=cid%> - query " + <%=query%>);
|
||||
<% } %>
|
||||
String dataset = query_<%=cid%>.substring(query_<%=cid%>.indexOf("from") + 4, query_<%=cid%>.indexOf(".")).trim();
|
||||
String temp_table = "temp_" + dataset + java.util.UUID.randomUUID().toString().replaceAll("-", "")
|
||||
+ "<%=cid%>".toLowerCase().replaceAll("[^a-z0-9]", "0").replaceAll("^[^a-z]", "a")
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
|
||||
|
||||
com.google.cloud.bigquery.BigQuery bigquery_<%=cid%> = com.google.cloud.bigquery.BigQueryOptions.newBuilder()
|
||||
.setCredentials(credentials_<%=cid%>)
|
||||
@@ -232,55 +233,25 @@
|
||||
.getService();
|
||||
|
||||
com.google.cloud.bigquery.QueryJobConfiguration.Builder queryConfiguration_<%=cid%> = com.google.cloud.bigquery.QueryJobConfiguration.newBuilder(<%=query%>).setUseLegacySql(<%=useLegacySql%>);
|
||||
|
||||
<%
|
||||
if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) {
|
||||
%>
|
||||
queryConfiguration_<%=cid%>.setDestinationTable(com.google.cloud.bigquery.TableId.of(dataset, temp_table)).setAllowLargeResults(true);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId.of(java.util.UUID.randomUUID().toString());
|
||||
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.newBuilder(queryConfiguration_<%=cid%>.build()).setJobId(jobId_<%=cid%>).build());
|
||||
|
||||
<%if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) {%>
|
||||
|
||||
com.google.cloud.bigquery.QueryJobConfiguration jobConfDryRun_<%=cid%> = queryConfiguration_<%=cid%> .setDryRun(true).build();
|
||||
com.google.cloud.bigquery.Job jobDryRun_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.of(jobConfDryRun_<%=cid%>));
|
||||
|
||||
String queryLocation_<%=cid%> =jobDryRun_<%=cid%>.getJobId().getLocation();
|
||||
String location_<%=cid%> = queryLocation_<%=cid%> == null ? "US" : queryLocation_<%=cid%>;
|
||||
String tempDataset_<%=cid%> = java.util.UUID.randomUUID().toString().replaceAll("-", "")
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt())
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
|
||||
String tempTable_<%=cid%> = java.util.UUID.randomUUID().toString().replaceAll("-", "")
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt())
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
|
||||
<% if(isLog4jEnabled){ %>
|
||||
log.info("<%=cid%> - query location :" + queryLocation_<%=cid%>);
|
||||
log.info("<%=cid%> - temporary dataset location :" + location_<%=cid%>);
|
||||
log.info("<%=cid%> - temporary Dataset name : " + tempDataset_<%=cid%>);
|
||||
log.info("<%=cid%> - temporary table name : " + tempTable_<%=cid%>);
|
||||
<% } %>
|
||||
|
||||
com.google.cloud.bigquery.DatasetInfo datasetInfo_<%=cid%> = com.google.cloud.bigquery.DatasetInfo.newBuilder(tempDataset_<%=cid%>).setLocation(location_<%=cid%>).build();
|
||||
com.google.cloud.bigquery.Dataset dataset_<%=cid%> = bigquery_<%=cid%> .create(datasetInfo_<%=cid%>);
|
||||
|
||||
queryConfiguration_<%=cid%>
|
||||
.setDryRun(false)
|
||||
.setAllowLargeResults(true)
|
||||
.setDestinationTable(com.google.cloud.bigquery.TableId.of(tempDataset_<%=cid%>, tempTable_<%=cid%>));
|
||||
|
||||
com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId
|
||||
.newBuilder().setProject(<%=projectId%>)
|
||||
.setJob(java.util.UUID.randomUUID().toString())
|
||||
.setLocation(location_<%=cid%>)
|
||||
.build();
|
||||
|
||||
<% if(isLog4jEnabled){ %>
|
||||
log.info("<%=cid%> - job location : " + jobId_<%=cid%>.getLocation());
|
||||
<% } %>
|
||||
<% } else { %>
|
||||
com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId.of(<%=projectId%>,java.util.UUID.randomUUID().toString());
|
||||
|
||||
<% } %>
|
||||
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.newBuilder(queryConfiguration_<%=cid%>.build()).setJobId(jobId_<%=cid%>).build());
|
||||
|
||||
|
||||
<% if(isLog4jEnabled) { %>
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Sending job " + jobId_<%=cid%> + " with query: " + <%=query%>);
|
||||
<% } %>
|
||||
|
||||
<%
|
||||
}
|
||||
%>
|
||||
job_<%=cid%> = job_<%=cid%>.waitFor();
|
||||
|
||||
if (job_<%=cid%> == null) {
|
||||
@@ -289,9 +260,13 @@
|
||||
throw new RuntimeException(job_<%=cid%>.getStatus().getError().toString());
|
||||
}
|
||||
|
||||
<% if(isLog4jEnabled) { %>
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Job " + jobId_<%=cid%> + " finished successfully.");
|
||||
<% } %>
|
||||
<%
|
||||
}
|
||||
%>
|
||||
com.google.cloud.bigquery.TableResult result_<%=cid%> = job_<%=cid%>.getQueryResults();
|
||||
long nb_line_<%=cid%> = 0;
|
||||
<%
|
||||
|
||||
@@ -10,10 +10,7 @@ imports="
|
||||
CodeGeneratorArgument codeGenArgument = (CodeGeneratorArgument) argument;
|
||||
INode node = (INode)codeGenArgument.getArgument();
|
||||
String cid = node.getUniqueName();
|
||||
String authMode = ElementParameterParser.getValue(node,"__AUTH_MODE__");
|
||||
String projectId = ElementParameterParser.getValue(node,"__PROJECT_ID__");
|
||||
String resultSizeType = ElementParameterParser.getValue(node,"__RESULT_SIZE__");
|
||||
|
||||
String authMode = ElementParameterParser.getValue(node,"__AUTH_MODE__");
|
||||
if (authMode.equals("OAUTH")) {
|
||||
%>
|
||||
}
|
||||
@@ -27,10 +24,7 @@ imports="
|
||||
} else if (authMode.equals("SERVICEACCOUNT")) {
|
||||
%>
|
||||
}
|
||||
<% if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) { %>
|
||||
com.google.cloud.bigquery.DatasetId datasetId_<%=cid%> = com.google.cloud.bigquery.DatasetId.of(<%=projectId%>, tempDataset_<%=cid%>);
|
||||
bigquery_<%=cid%>.delete(datasetId_<%=cid%>, com.google.cloud.bigquery.BigQuery.DatasetDeleteOption.deleteContents());
|
||||
<% }%>
|
||||
bigquery_<%=cid%>.delete(com.google.cloud.bigquery.TableId.of(dataset, temp_table));
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
|
||||
@@ -188,7 +188,6 @@ COMPATIBILITY="ALL"
|
||||
<IMPORT NAME="sshj-0_8_1" MODULE="sshj-0.8.1.jar" MVN="mvn:org.talend.libraries/sshj-0.8.1/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
|
||||
<IMPORT NAME="sts-2.0.2" MODULE="sts-2.0.2.jar" MVN="mvn:org.talend.libraries/sts-2.0.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="jaxb-api-2.3.1" MODULE="jaxb-api-2.3.1.jar" MVN="mvn:javax.xml.bind/jaxb-api/2.3.1" REQUIRED="true"/>
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -97,7 +97,6 @@ COMPATIBILITY="ALL"
|
||||
<IMPORT NAME="sshj-0_8_1" MODULE="sshj-0.8.1.jar" MVN="mvn:org.talend.libraries/sshj-0.8.1/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
|
||||
<IMPORT NAME="sts-2.0.2" MODULE="sts-2.0.2.jar" MVN="mvn:org.talend.libraries/sts-2.0.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="jaxb-api-2.3.1" MODULE="jaxb-api-2.3.1.jar" MVN="mvn:javax.xml.bind/jaxb-api/2.3.1" REQUIRED="true"/>
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -53,7 +53,6 @@
|
||||
|
||||
<PARAMETER
|
||||
NAME="DBTYPE"
|
||||
REPOSITORY_VALUE="DBTYPE"
|
||||
FIELD="CLOSED_LIST"
|
||||
NUM_ROW="10"
|
||||
>
|
||||
@@ -898,8 +897,8 @@
|
||||
<IMPORT NAME="Driver-VERTICA_7.0.1" MODULE="vertica-jdbc-7.0.1-0.jar" MVN="mvn:org.talend.libraries/vertica-jdbc-7.0.1-0/6.0.0" REQUIRED_IF="(DBTYPE=='VERTICA') and (USE_EXISTING_CONNECTION == 'false') and (DB_VERTICA_VERSION=='VERTICA_7_0_X')" />
|
||||
<IMPORT NAME="Driver-VERTICA_7.1.2" MODULE="vertica-jdbc-7.1.2-0.jar" MVN="mvn:org.talend.libraries/vertica-jdbc-7.1.2-0/6.0.0" REQUIRED_IF="(DBTYPE=='VERTICA') and (USE_EXISTING_CONNECTION == 'false') and (DB_VERTICA_VERSION=='VERTICA_7_1_X')" />
|
||||
<IMPORT NAME="Driver-EXASolution" MODULE="exajdbc-6.0.9302.jar" MVN="mvn:org.talend.libraries/exajdbc-6.0.9302/6.3.0" REQUIRED_IF="(DBTYPE=='EXASOL') and (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-Redshift" MODULE="redshift-jdbc42-no-awssdk-1.2.32.1056.jar" MVN="mvn:com.amazon.redshift/redshift-jdbc42-no-awssdk/1.2.32.1056" REQUIRED_IF="(DBTYPE=='REDSHIFT') and (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-Snowflake" MODULE="snowflake-jdbc-3.9.0.jar" MVN="mvn:net.snowflake/snowflake-jdbc/3.9.0" REQUIRED_IF="(DBTYPE=='SNOWFLAKE') and (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-Redshift" MODULE="redshift-jdbc42-no-awssdk-1.2.20.1043.jar" MVN="mvn:com.amazon.redshift/redshift-jdbc42-no-awssdk/1.2.20.1043" REQUIRED_IF="(DBTYPE=='REDSHIFT') and (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-Snowflake" MODULE="snowflake-jdbc-3.5.3.jar" MVN="mvn:net.snowflake/snowflake-jdbc/3.5.3" REQUIRED_IF="(DBTYPE=='SNOWFLAKE') and (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-VERTICA_9.0" MODULE="vertica-jdbc-9.0.0-0.jar" MVN="mvn:org.talend.libraries/vertica-jdbc-9.0.0-0/6.0.0" REQUIRED_IF="(DBTYPE=='VERTICA') and (USE_EXISTING_CONNECTION == 'false') and (DB_VERTICA_VERSION=='VERTICA_9_0')"/>
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -1564,7 +1564,7 @@ class TeradataManager extends Manager {
|
||||
private boolean useExistingConnection;
|
||||
private String connection;
|
||||
|
||||
private static final String SEPARATOR = "/";
|
||||
private static final String SEPARATOR = ",";
|
||||
|
||||
protected TeradataManager(String host, String port, String dbName, String tableName,
|
||||
String userName, String cid, boolean useExistingConnection, String connection, String additionalParams ) {
|
||||
|
||||
@@ -118,7 +118,7 @@ DB_VERTICA_VERSION.ITEM.VERTICA_7_1_X=Vertica 7.1.X
|
||||
DB_VERTICA_VERSION.ITEM.VERTICA_9_0=Vertica 9
|
||||
|
||||
DB_POSTGRE_VERSION.NAME=DB Version
|
||||
DB_POSTGRE_VERSION.ITEM.PRIOR_TO_V9=v7.2 to v8.X
|
||||
DB_POSTGRE_VERSION.ITEM.PRIOR_TO_V9=Prior to v9
|
||||
DB_POSTGRE_VERSION.ITEM.V9_X=v9 and later
|
||||
QUERY.NAME=Query
|
||||
|
||||
|
||||
@@ -40,8 +40,8 @@
|
||||
</ADVANCED_PARAMETERS>
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT MODULE="janet-java-1.2.jar" MVN="mvn:org.talend.libraries/janet-java-1.2/6.0.0" NAME="janet-java-1.2"
|
||||
REQUIRED="true" />
|
||||
<IMPORT MODULE="janet-java-1.1.jar" MVN="mvn:org.talend.libraries/janet-java-1.1/6.0.0" NAME="janet-java-1.1"
|
||||
REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS>
|
||||
|
||||
@@ -94,7 +94,7 @@
|
||||
</ADVANCED_PARAMETERS>
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT MODULE="janet-java-1.2.jar" MVN="mvn:org.talend.libraries/janet-java-1.2/6.0.0" NAME="janet-java-1.2"
|
||||
<IMPORT MODULE="janet-java-1.1.jar" MVN="mvn:org.talend.libraries/janet-java-1.1/6.0.0" NAME="janet-java-1.1"
|
||||
REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -36,8 +36,6 @@ skeleton="@{org.talend.designer.components.localprovider}/components/templates/d
|
||||
String defaultDateFormat = ElementParameterParser.getValue(node, "__DEFAULT_DATE_FORMAT__");
|
||||
String defaultTimestampFormat = ElementParameterParser.getValue(node, "__DEFAULT_TIMESTAMP_FORMAT__");
|
||||
String numGroupSep = ElementParameterParser.getValue(node, "__THOUSANDS_SEPARATOR__");
|
||||
boolean useLenPrecision = "true".equals(ElementParameterParser.getValue(node, "__USE_PRECISION_LENGTH_FROM_SCHEMA__"));
|
||||
|
||||
if (numGroupSep == null|| numGroupSep.trim().isEmpty()) {
|
||||
numGroupSep = "null";
|
||||
}
|
||||
@@ -104,6 +102,7 @@ skeleton="@{org.talend.designer.components.localprovider}/components/templates/d
|
||||
|
||||
int deletedCount_<%=cid%> = 0;
|
||||
int insertedCount_<%=cid%> = 0;
|
||||
// schema is mandatory
|
||||
|
||||
String dbschema_<%=cid%> = null;
|
||||
<% if (useExistingConnection) { %>
|
||||
@@ -111,12 +110,8 @@ String dbschema_<%=cid%> = null;
|
||||
<% } else { %>
|
||||
dbschema_<%=cid%> = <%=dbschema%>;
|
||||
<% } %>
|
||||
String tableName_<%=cid%> = null;
|
||||
if(dbschema_<%=cid%> == null || dbschema_<%=cid%>.trim().length() == 0) {
|
||||
tableName_<%=cid%> = <%=table%>;
|
||||
} else {
|
||||
tableName_<%=cid%> = dbschema_<%=cid%> + "." + <%=table%>;
|
||||
}
|
||||
|
||||
String tableName_<%=cid%> = dbschema_<%=cid%> + "." + <%=table%>;
|
||||
java.sql.Connection conn_<%=cid%> = null;
|
||||
<% if (useExistingConnection) { %>
|
||||
conn_<%=cid%> = (java.sql.Connection) globalMap.get("conn_<%=connection%>");
|
||||
@@ -212,24 +207,12 @@ org.talend.database.exasol.imp.EXABulkUtil <%=cid%> = new org.talend.database.ex
|
||||
String lengthStr = "null";
|
||||
String precisionStr = "null";
|
||||
Integer length = column.getLength();
|
||||
if(useLenPrecision) {
|
||||
if (length != null) {
|
||||
lengthStr = String.valueOf(length);
|
||||
}
|
||||
} else {
|
||||
if (length != null && hasGroupSep) {
|
||||
lengthStr = String.valueOf(length);
|
||||
}
|
||||
}
|
||||
if (length != null) {
|
||||
lengthStr = String.valueOf(length);
|
||||
}
|
||||
Integer precision = column.getPrecision();
|
||||
if(useLenPrecision) {
|
||||
if (precision != null) {
|
||||
precisionStr = String.valueOf(precision);
|
||||
}
|
||||
} else {
|
||||
if (precision != null && hasGroupSep) {
|
||||
precisionStr = String.valueOf(precision);
|
||||
}
|
||||
if (precision != null) {
|
||||
precisionStr = String.valueOf(precision);
|
||||
}
|
||||
String talendType = column.getTalendType();
|
||||
if ("id_Date".equals(talendType)
|
||||
|
||||
@@ -635,23 +635,14 @@
|
||||
>
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="USE_PRECISION_LENGTH_FROM_SCHEMA"
|
||||
FIELD="CHECK"
|
||||
NUM_ROW="115"
|
||||
REQUIRED="true"
|
||||
>
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
</ADVANCED_PARAMETERS>
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-EXASolution" MODULE="exajdbc-6.0.9302.jar" MVN="mvn:org.talend.libraries/exajdbc-6.0.9302/6.3.0" REQUIRED="true" />
|
||||
<IMPORT NAME="Talend-DB-Exasol-Util" MODULE="talend-db-exasol-2.1.4.jar" MVN="mvn:org.talend.components.lib/talend-db-exasol/2.1.4"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.jdbc.exasol/lib/talend-db-exasol-2.1.4.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="Talend-DB-Exasol-Util" MODULE="talend-db-exasol-2.1.2.jar" MVN="mvn:org.talend.libraries/talend-db-exasol-2.1.2/6.2.0"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.jdbc.exasol/lib/talend-db-exasol-2.1.2.jar" REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -108,6 +108,4 @@ NB_LINE_INSERTED.NAME=Rows inserted
|
||||
NB_LINE_DELETED.NAME=Rows deleted
|
||||
SOURCE_QUERY.NAME=Source query
|
||||
|
||||
TEST_MODE.NAME=Test mode (no statements are executed)
|
||||
|
||||
USE_PRECISION_LENGTH_FROM_SCHEMA.NAME=Use precision and length from schema
|
||||
TEST_MODE.NAME=Test mode (no statements are executed)
|
||||
@@ -105,12 +105,8 @@ if(("true").equals(useExistingConn)) {
|
||||
<%
|
||||
}
|
||||
%>
|
||||
String tableName_<%=cid%> = null;
|
||||
if(dbschema_<%=cid%> == null || dbschema_<%=cid%>.trim().length() == 0) {
|
||||
tableName_<%=cid%> = <%=table%>;
|
||||
} else {
|
||||
tableName_<%=cid%> = dbschema_<%=cid%> + "." + <%=table%>;
|
||||
}
|
||||
// schema is mandatory
|
||||
String tableName_<%=cid%> = dbschema_<%=cid%> + "." + <%=table%>;
|
||||
<%
|
||||
if(("UPDATE").equals(dataAction) || ("INSERT_OR_UPDATE").equals(dataAction) || ("UPDATE_OR_INSERT").equals(dataAction)) {
|
||||
int updateKeyCount = 0;
|
||||
|
||||
@@ -178,7 +178,7 @@
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Java_DOM4J1.6" MODULE="dom4j-1.6.1.jar" MVN="mvn:dom4j/dom4j/1.6.1" UrlPath="platform:/plugin/org.talend.libraries.dom4j-jaxen/lib/dom4j-1.6.1.jar" REQUIRED_IF="READ_BY == 'XPATH'" BundleID="" />
|
||||
<IMPORT NAME="Java_JAXEN1.1" MODULE="jaxen-1.1.1.jar" MVN="mvn:org.talend.libraries/jaxen-1.1.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.dom4j-jaxen/lib/jaxen-1.1.1.jar" REQUIRED_IF="READ_BY == 'XPATH'" BundleID="" />
|
||||
<IMPORT NAME="json-lib" MODULE="json-lib-2.4.3-talend.jar" MVN="mvn:net.sf.json-lib/json-lib/2.4.3-talend" REQUIRED_IF="READ_BY == 'XPATH'" />
|
||||
<IMPORT NAME="json-lib" MODULE="json-lib-2.4.1-talend.jar" MVN="mvn:net.sf.json-lib/json-lib/2.4.1-talend" REQUIRED_IF="READ_BY == 'XPATH'" />
|
||||
<IMPORT NAME="commons_lang" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED_IF="READ_BY == 'XPATH'" />
|
||||
<IMPORT NAME="commons_logging" MODULE="commons-logging-1.1.1.jar" MVN="mvn:org.talend.libraries/commons-logging-1.1.1/6.0.0" UrlPath="platform:/base/plugins/org.apache.commons.logging_1.1.1.v201101211721.jar" REQUIRED_IF="READ_BY == 'XPATH'" />
|
||||
<IMPORT NAME="ezmorph" MODULE="ezmorph-1.0.6.jar" MVN="mvn:org.talend.libraries/ezmorph-1.0.6/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.jackson/lib/ezmorph-1.0.6.jar" REQUIRED_IF="READ_BY == 'XPATH'" />
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
<%@ jet
|
||||
%>
|
||||
|
||||
boolean retry_<%=cid%> = false;
|
||||
int retry_count_<%=cid%> = 0;
|
||||
int retry_max_<%=cid%> = 5;
|
||||
|
||||
com.jcraft.jsch.Session session_<%=cid%> = null;
|
||||
com.jcraft.jsch.Channel channel_<%=cid%> = null;
|
||||
|
||||
do {
|
||||
retry_<%=cid%> = false;
|
||||
|
||||
com.jcraft.jsch.JSch jsch_<%=cid%> = new com.jcraft.jsch.JSch();
|
||||
|
||||
<%if (("PUBLICKEY").equals(authMethod)){%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - SFTP authentication using a public key.");
|
||||
log.debug("<%=cid%> - Private key: '" + <%=privateKey%> + "'.");
|
||||
<%}%>
|
||||
jsch_<%=cid%>.addIdentity(<%=privateKey %>, defaultUserInfo_<%=cid%>.getPassphrase());
|
||||
<%}%>
|
||||
|
||||
session_<%=cid%> = jsch_<%=cid%>.getSession(<%=user%>, <%=host%>, <%=port%>);
|
||||
session_<%=cid%>.setConfig("PreferredAuthentications", "publickey,password,keyboard-interactive,gssapi-with-mic");
|
||||
|
||||
<%if (("PASSWORD").equals(authMethod)) {%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - SFTP authentication using a password.");
|
||||
<%}%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
session_<%=cid%>.setPassword(decryptedPassword_<%=cid%>);
|
||||
<%}%>
|
||||
|
||||
session_<%=cid%>.setUserInfo(defaultUserInfo_<%=cid%>);
|
||||
<%if (!useProxy) {%>
|
||||
if(("true").equals(System.getProperty("http.proxySet")) ){
|
||||
com.jcraft.jsch.ProxyHTTP proxy_<%=cid%> = new com.jcraft.jsch.ProxyHTTP(System.getProperty("http.proxyHost"),Integer.parseInt(System.getProperty("http.proxyPort")));
|
||||
if(!"".equals(System.getProperty("http.proxyUser"))){
|
||||
proxy_<%=cid%>.setUserPasswd(System.getProperty("http.proxyUser"),System.getProperty("http.proxyPassword"));
|
||||
}
|
||||
session_<%=cid%>.setProxy(proxy_<%=cid%>);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user%> + "'.");
|
||||
<%}%>
|
||||
|
||||
channel_<%=cid%> = null;
|
||||
try {
|
||||
session_<%=cid%>.connect();
|
||||
channel_<%=cid%> = session_<%=cid%>.openChannel("sftp");
|
||||
channel_<%=cid%>.connect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
} catch (com.jcraft.jsch.JSchException e_<%=cid%>) {
|
||||
try {
|
||||
if(channel_<%=cid%>!=null) {
|
||||
channel_<%=cid%>.disconnect();
|
||||
}
|
||||
|
||||
if(session_<%=cid%>!=null) {
|
||||
session_<%=cid%>.disconnect();
|
||||
}
|
||||
} catch(java.lang.Exception ce_<%=cid%>) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - close sftp connection failed : " + ce_<%=cid%>.getClass() + " : " + ce_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
}
|
||||
|
||||
String message_<%=cid%> = new TalendException(null, null, null).getExceptionCauseMessage(e_<%=cid%>);
|
||||
if(message_<%=cid%>.contains("Signature length not correct") || message_<%=cid%>.contains("connection is closed by foreign host")) {
|
||||
retry_<%=cid%> = true;
|
||||
retry_count_<%=cid%>++;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - connect: Signature length not correct or connection is closed by foreign host, so retry, retry time : " + retry_count_<%=cid%>);
|
||||
<%}%>
|
||||
} else {
|
||||
throw e_<%=cid%>;
|
||||
}
|
||||
}
|
||||
} while(retry_<%=cid%> && (retry_count_<%=cid%> < retry_max_<%=cid%>));
|
||||
|
||||
com.jcraft.jsch.ChannelSftp c_<%=cid%> = (com.jcraft.jsch.ChannelSftp)channel_<%=cid%>;
|
||||
@@ -28,11 +28,10 @@ String authMethod = ElementParameterParser.getValue(node, "__AUTH_METHOD__");
|
||||
String privateKey = ElementParameterParser.getValue(node, "__PRIVATEKEY__");
|
||||
boolean ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String encoding = ElementParameterParser.getValue(node, "__ENCODING__");
|
||||
String sEncoding = ElementParameterParser.getValue(node, "__ENCODING__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
String protectionLevel = ElementParameterParser.getValue(node, "__FTPS_PROT__");
|
||||
String protectionBufferSize = ElementParameterParser.getValue(node, "__FTPS_PROTECTION_BUFF_SIZE__");
|
||||
String timeoutValue = ElementParameterParser.getValue(node, "__CONNECTION_TIMEOUT__");
|
||||
boolean useRemoteVerification = ("true").equals(ElementParameterParser.getValue(node, "__REMOTE_VERIFICATION__"));
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
String passwordFieldName = "";
|
||||
@@ -61,18 +60,37 @@ if (ElementParameterParser.canEncrypt(node, passwordFieldName)) {
|
||||
});
|
||||
<%
|
||||
}
|
||||
%>
|
||||
int connectionTimeout_<%=cid %> = Integer.valueOf(<%=timeoutValue %>);
|
||||
<%
|
||||
if (!sftp && !ftps) { // *** ftp *** //
|
||||
%>
|
||||
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> = null;
|
||||
<%
|
||||
passwordFieldName = "__PASS__";
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
|
||||
|
||||
globalMap.put("conn_<%=cid%>",ftp_<%=cid %>);
|
||||
if (!sftp && !ftps) { // *** ftp *** //
|
||||
if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
|
||||
<%}%>
|
||||
com.enterprisedt.net.ftp.FTPClient ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
|
||||
ftp_<%=cid %>.setRemoteHost(<%=host %>);
|
||||
ftp_<%=cid %>.setRemotePort(<%=port %>);
|
||||
|
||||
<%if (("ACTIVE").equals(connectMode)) {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
|
||||
<%} else {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setControlEncoding(<%=sEncoding%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" +<%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect();
|
||||
|
||||
<%
|
||||
passwordFieldName = "__PASS__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
globalMap.put("conn_<%=cid%>",ftp_<%=cid %>);
|
||||
<%
|
||||
} else if (!ftps) { // *** sftp *** //
|
||||
%>
|
||||
@@ -120,20 +138,99 @@ if (!sftp && !ftps) { // *** ftp *** //
|
||||
passwordFieldName = "__PASS__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
|
||||
<%@ include file="sftp_connect.javajet"%>
|
||||
|
||||
<%if(use_encoding) {%>
|
||||
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
|
||||
c_<%=cid%>.setFilenameEncoding(<%=sEncoding%>);
|
||||
<%}%>
|
||||
|
||||
globalMap.put("conn_<%=cid%>", c_<%=cid%>);
|
||||
<%
|
||||
} else { // *** ftps *** //
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
|
||||
globalMap.put("conn_<%=cid%>", ftp_<%=cid %>);
|
||||
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
|
||||
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
|
||||
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
|
||||
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
MyTrust_<%=cid%> myTrust_<%=cid%> = null;
|
||||
|
||||
try {
|
||||
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
|
||||
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
|
||||
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
|
||||
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
|
||||
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
|
||||
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
|
||||
if (isHTTPProxyUsed) {
|
||||
String httpProxyHost = System.getProperty("http.proxyHost");
|
||||
int httpProxyPort = Integer.getInteger("http.proxyPort");
|
||||
String httpProxyUser = System.getProperty("http.proxyUser");
|
||||
String httpProxyPass = System.getProperty("http.proxyPassword");
|
||||
|
||||
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
|
||||
} else {
|
||||
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
|
||||
}
|
||||
ftp_<%=cid %>.setControlEncoding(<%=sEncoding %>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
|
||||
<%
|
||||
passwordFieldName = "__PASS__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
|
||||
if (!isLoginSuccessful_<%=cid %>) {
|
||||
throw new RuntimeException("Login failed");
|
||||
}
|
||||
|
||||
ftp_<%=cid %>.setFileType(org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE);
|
||||
<%
|
||||
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
|
||||
<%
|
||||
}
|
||||
|
||||
if (protectionLevel != null && !protectionLevel.isEmpty()) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} catch (Exception e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
|
||||
<%}%>
|
||||
throw e;
|
||||
}
|
||||
|
||||
<%
|
||||
if ("PASSIVE".equals(connectMode)) {
|
||||
%>
|
||||
ftp_<%=cid %>.enterLocalPassiveMode();
|
||||
<%
|
||||
if (isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - Using the passive mode.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
globalMap.put("conn_<%=cid%>", ftp_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -232,11 +232,6 @@
|
||||
NUM_ROW="95" SHOW_IF="(FTPS=='true')" REPOSITORY_VALUE="REMOTE_VERIFICATION">
|
||||
<DEFAULT>true</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="CONNECTION_TIMEOUT" FIELD="TEXT"
|
||||
NUM_ROW="96" REPOSITORY_VALUE="CONNECTION_TIMEOUT">
|
||||
<DEFAULT>0</DEFAULT>
|
||||
</PARAMETER>
|
||||
</ADVANCED_PARAMETERS>
|
||||
|
||||
<CODEGENERATION>
|
||||
@@ -244,7 +239,7 @@
|
||||
<IMPORT NAME="Java-FTP" MODULE="edtftpj-2.5.0.jar" MVN="mvn:com.enterprisedt/edtftpj/2.5.0" UrlPath="platform:/plugin/org.talend.libraries.ftp/lib/edtftpj-2.5.0.jar" REQUIRED_IF="(SFTP == 'false') AND (FTPS == 'false')"/>
|
||||
<IMPORT NAME="Java-SFTP" MODULE="jsch-0.1.55.jar" MVN="mvn:com.jcraft/jsch/0.1.55" REQUIRED_IF="(SFTP == 'true')"/>
|
||||
<IMPORT NAME="Java-FTPS" MODULE="commons-net-3.6.jar" MVN="mvn:commons-net/commons-net/3.6" REQUIRED_IF="(FTPS == 'true')" />
|
||||
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190819.jar" MVN="mvn:org.talend.components.lib/commons-net-ftps-proxy/3.6.1-talend-20190819" REQUIRED_IF="(FTPS == 'true')" />
|
||||
<IMPORT NAME="Java-FTPS-talend" MODULE="commons-net-ftps-proxy-3.6.1-talend-20190128.jar" MVN="mvn:org.talend.libraries/commons-net-ftps-proxy/3.6.1-talend-20190128" REQUIRED_IF="(FTPS == 'true')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -40,5 +40,4 @@ SECURITY_MODE.NAME=Security Mode
|
||||
SECURITY_MODE.ITEM.IMPLICIT=Implicit
|
||||
SECURITY_MODE.ITEM.EXPLICIT=Explicit
|
||||
|
||||
USE_ENCODING.NAME=Filename encoding
|
||||
CONNECTION_TIMEOUT.NAME=Connection timeout (milliseconds)
|
||||
USE_ENCODING.NAME=Filename encoding
|
||||
@@ -43,7 +43,6 @@ boolean sftp = false;
|
||||
boolean ftps = false;
|
||||
String protectionLevel = ElementParameterParser.getValue(node, "__FTPS_PROT__");
|
||||
String protectionBufferSize = ElementParameterParser.getValue(node, "__FTPS_PROTECTION_BUFF_SIZE__");
|
||||
String timeoutValue = ElementParameterParser.getValue(node, "__CONNECTION_TIMEOUT__");
|
||||
boolean useRemoteVerification = ("true").equals(ElementParameterParser.getValue(node, "__REMOTE_VERIFICATION__"));
|
||||
if (("true").equals(useExistingConn)) {
|
||||
List<? extends INode> nodeList = node.getProcess().getGeneratingNodes();
|
||||
@@ -56,9 +55,6 @@ if (("true").equals(useExistingConn)) {
|
||||
} else {
|
||||
sftp = ("true").equals(ElementParameterParser.getValue(node, "__SFTP__"));
|
||||
ftps = ("true").equals(ElementParameterParser.getValue(node, "__FTPS__"));
|
||||
%>
|
||||
int connectionTimeout_<%=cid %> = Integer.valueOf(<%=timeoutValue %>);
|
||||
<%
|
||||
}
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
@@ -133,7 +129,7 @@ int nb_file_<%=cid%> = 0;
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/sftp_connect.javajet"%>
|
||||
<%@ include file="../tFTPConnection/sftp_connect.javajet"%>
|
||||
|
||||
<%if(use_encoding) {%>
|
||||
c_<%=cid%>.setFilenameEncoding(<%=encoding%>);
|
||||
@@ -171,18 +167,101 @@ int nb_file_<%=cid%> = 0;
|
||||
|
||||
<%
|
||||
} else if (ftps) { // *** ftps *** //
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = null;
|
||||
<%
|
||||
if (("false").equals(useExistingConn)) {
|
||||
String keystoreFile = ElementParameterParser.getValue(node, "__KEYSTORE_FILE__");
|
||||
String securityMode = ElementParameterParser.getValue(node, "__SECURITY_MODE__");
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftps_connect.javajet"%>
|
||||
<%@ include file="../templates/FTP/ftpsTrust.javajet"%>
|
||||
javax.net.ssl.SSLContext sslContext_<%=cid %> = null;
|
||||
javax.net.ssl.TrustManager[] trustManager_<%=cid %> = null;
|
||||
javax.net.ssl.SSLSocketFactory sslSocketFactory_<%=cid %> = null;
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> =null;
|
||||
MyTrust_<%=cid %> myTrust_<%=cid%> = null;
|
||||
|
||||
try {
|
||||
sslContext_<%=cid %> = javax.net.ssl.SSLContext.getInstance("SSL");
|
||||
myTrust_<%=cid%> = new MyTrust_<%=cid%>();
|
||||
trustManager_<%=cid %> = myTrust_<%=cid%>.getTrustManagers();
|
||||
sslContext_<%=cid %>.init(null, trustManager_<%=cid %>, new java.security.SecureRandom());
|
||||
sslSocketFactory_<%=cid %> = sslContext_<%=cid %>.getSocketFactory();
|
||||
boolean isHTTPProxyUsed = "true".equals(System.getProperty("http.proxySet"));
|
||||
if (isHTTPProxyUsed) {
|
||||
String httpProxyHost = System.getProperty("http.proxyHost");
|
||||
int httpProxyPort = Integer.getInteger("http.proxyPort");
|
||||
String httpProxyUser = System.getProperty("http.proxyUser");
|
||||
String httpProxyPass = System.getProperty("http.proxyPassword");
|
||||
|
||||
ftp_<%=cid %> = new org.talend.ftp.HTTPProxyFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>, httpProxyHost, httpProxyPort, httpProxyUser, httpProxyPass);
|
||||
} else {
|
||||
ftp_<%=cid %> = new org.talend.ftp.SSLSessionReuseFTPSClient(<%="IMPLICIT".equals(securityMode)%>, sslContext_<%=cid %>);
|
||||
}
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> -FTPS security Mode is <%=securityMode%>.");
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username '" + <%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect(<%=host %>,<%=ftpsPort %>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setRemoteVerificationEnabled(<%=useRemoteVerification %>);
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
boolean isLoginSuccessful_<%=cid %> = ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
|
||||
if (!isLoginSuccessful_<%=cid %>) {
|
||||
throw new RuntimeException("Login failed");
|
||||
}
|
||||
|
||||
<%
|
||||
if (protectionBufferSize != null && !protectionBufferSize.isEmpty() ) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPBSZ(<%=protectionBufferSize %>);
|
||||
<%
|
||||
}
|
||||
|
||||
if (protectionLevel != null && !protectionLevel.isEmpty()) {
|
||||
%>
|
||||
ftp_<%=cid %>.execPROT(<%=protectionLevel %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} catch (Exception e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - Can't create connection: " + e.getMessage());
|
||||
<%}%>
|
||||
<%
|
||||
if (dieOnError) {
|
||||
%>
|
||||
throw e;
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
e.printStackTrace();
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
|
||||
<%
|
||||
if ("PASSIVE".equals(connectMode)) {
|
||||
%>
|
||||
ftp_<%=cid %>.enterLocalPassiveMode();
|
||||
<%
|
||||
if (isLog4jEnabled) {
|
||||
%>
|
||||
log.debug("<%=cid%> - Using the passive mode.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
org.apache.commons.net.ftp.FTPSClient ftp_<%=cid %> = (org.apache.commons.net.ftp.FTPSClient) globalMap.get("<%=conn %>");
|
||||
<%
|
||||
if (moveToCurrentDir) {
|
||||
%>
|
||||
@@ -224,12 +303,36 @@ int nb_file_<%=cid%> = 0;
|
||||
<%if(!moveToCurrentDir){%>
|
||||
String rootDir_<%=cid %> = ftp_<%=cid %>.pwd();
|
||||
<%}%>
|
||||
<%} else {
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/FTP/ftp_connect.javajet"%>
|
||||
<%}%>
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
|
||||
<%} else {%>
|
||||
ftp_<%=cid %> = new com.enterprisedt.net.ftp.FTPClient();
|
||||
ftp_<%=cid %>.setRemoteHost(<%=host %>);
|
||||
ftp_<%=cid %>.setRemotePort(<%=port %>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - " + <%=(("ACTIVE").equals(connectMode))?"\"Using the active mode.\"":"\"Using the passive mode.\""%>);
|
||||
<%}%>
|
||||
<%if (("ACTIVE").equals(connectMode)) {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.ACTIVE);
|
||||
<%} else {%>
|
||||
ftp_<%=cid %>.setConnectMode(com.enterprisedt.net.ftp.FTPConnectMode.PASV);
|
||||
<%}%>
|
||||
ftp_<%=cid %>.setControlEncoding(<%=encoding%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Attempt to connect to '" + <%=host %> + "' with username: '" +<%=user %>+ "'.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.connect();
|
||||
|
||||
<%
|
||||
passwordFieldName = "__PASSWORD__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
ftp_<%=cid %>.login(<%=user %>, decryptedPassword_<%=cid%>);
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connect to '" + <%=host %> + "' has succeeded.");
|
||||
<%}%>
|
||||
<%}%>
|
||||
java.util.List<java.util.Map<String,String>> list<%=cid %> = new java.util.ArrayList<java.util.Map<String,String>>();
|
||||
|
||||
<%for (int i = 0; i < files.size(); i++) {
|
||||
Map<String, String> line = files.get(i);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user