Compare commits
43 Commits
allDeltaAn
...
patch/7.3.
| Author | SHA1 | Date | |
|---|---|---|---|
|
5c02590cc0 | ||
|
|
77c2f9f4bd | ||
|
|
7366be4678 | ||
|
|
a9c13a8ecc | ||
|
|
9c3d6e913d | ||
|
|
d5648f978c | ||
|
|
98d0a0faea | ||
|
|
73fc573d77 | ||
|
|
8e53443f64 | ||
|
|
7ca4e0a7a6 | ||
|
|
c8678c0b13 | ||
|
|
2ef88b756c | ||
|
|
781b8f587e | ||
|
|
00c9f5c0df | ||
|
|
cad0a7cc1f | ||
|
|
6e301c26f6 | ||
|
|
2e87b46081 | ||
|
|
a263ca4769 | ||
|
|
e145d170bc | ||
|
|
ba146ef1ac | ||
|
|
2047dee494 | ||
|
|
faaac52ba7 | ||
|
|
b4515a9778 | ||
|
|
f76ff12ded | ||
|
|
72f1b4998b | ||
|
|
e3224a1628 | ||
|
|
6bafdb03aa | ||
|
|
f937a8b27d | ||
|
|
9454265a67 | ||
|
|
a9e7d7b09a | ||
|
|
afb079f6f3 | ||
|
|
0294795b51 | ||
|
|
f5cc930f5f | ||
|
|
f7e0e22d0b | ||
|
|
54bfc6bb81 | ||
|
|
5d81348f41 | ||
|
|
7fd431d8cb | ||
|
|
0eef8c764e | ||
|
|
c959cc95ce | ||
|
|
80dbe96346 | ||
|
|
bcc4abeb64 | ||
|
|
908baade0b | ||
|
|
8aa65ca92b |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -11,4 +11,3 @@ main/plugins/org.talend.designer.codegen/resources/sparkstreaming_header.javajet
|
||||
main/plugins/org.talend.designer.codegen/resources/sparkstreaming_subprocess_footer.javajet
|
||||
main/plugins/org.talend.designer.codegen/resources/sparkstreaming_subprocess_header.javajet
|
||||
main/plugins/org.talend.designer.codegen/resources/routelet.skeleton
|
||||
.DS_Store
|
||||
@@ -4,6 +4,5 @@ Huge thanks to these people who contributed in making Talend product better:
|
||||
- [jlolling](https://github.com/jlolling) (Jan -and Miss- Lolling)
|
||||
- [kirkalicious](https://github.com/kirkalicious) (Kirk Tarou)
|
||||
- [mrizzato](https://github.com/mrizzato) (Massimiliano Rizzato)
|
||||
- [overpackaging](https://github.com/overpackaging) (Steeve Dreux)
|
||||
- [brazabr](https://github.com/brazabr) (Thiago Benvenuto)
|
||||
- [phoflack](https://jira.talendforge.org/secure/ViewProfile.jspa?name=phoflack) (Philip Hoflack)
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.abstractmap.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.alfrescooutput.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.business.diagram.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.codegen.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.components.exchange.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.core.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.dbmap.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.documentation.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.fileoutputxml.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.gefabstractmap.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.mapper.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.maven.job.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.rowgenerator.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.runprocess.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.scd.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.webservice.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.xmlmap.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.expressionbuilder.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.presentation.onboarding.resource.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.presentation.onboarding.resource.tos.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.repository.ftp.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.repository.json.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.repository.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.repository.view.di.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.sbi.engines.client.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.spagic.engines.client.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.sqlbuilder.nl</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.business.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.fragment.jars.di.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.help.tos.components.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.rcp.branding.tos.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.rcp.branding.tos.test.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.repository.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.sbi.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.spagic.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.tos.components.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.tos.core.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.tos.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.tos.libraries.feature</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.abstractmap</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.alfrescooutput</artifactId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.business.diagram</artifactId>
|
||||
|
||||
@@ -86,18 +86,11 @@ if((codePart.equals(ECodePart.END))&&(stat || logstashCurrent)){
|
||||
for(IConnection con:connSet){
|
||||
INode source = con.getSource();
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
%>
|
||||
if(runStat.updateStatAndLog(execStat,enableLogStash,resourceMap,iterateId,"<%=con.getUniqueName()%>",2,0,
|
||||
"<%=sourceNodeId%>","<%=sourceNodeLabel%>","<%=sourceNodeComponent%>","<%=node.getUniqueName()%>","<%=targetNodeLabel%>","<%=node.getComponent().getName()%>","<%="REJECT".equals(con.getConnectorName()) ? "reject" : "output"%>")) {
|
||||
<%=jobStructureCatcher.getUniqueName()%>,"<%=sourceNodeId%>","<%=sourceNodeComponent%>","<%=node.getUniqueName()%>","<%=node.getComponent().getName()%>","<%="REJECT".equals(con.getConnectorName()) ? "reject" : "output"%>")) {
|
||||
<%=jobStructureCatcher.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
}
|
||||
<%
|
||||
@@ -120,18 +113,11 @@ if((codePart.equals(ECodePart.END))&&(stat || logstashCurrent)){
|
||||
for(IConnection con:connSet){
|
||||
INode source = con.getSource();
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
%>
|
||||
if(runStat.log(resourceMap,iterateId,"<%=con.getUniqueName()%>",2,0,
|
||||
"<%=sourceNodeId%>","<%=sourceNodeLabel%>","<%=sourceNodeComponent%>","<%=node.getUniqueName()%>","<%=targetNodeLabel%>","<%=node.getComponent().getName()%>","<%="REJECT".equals(con.getConnectorName()) ? "reject" : "output"%>")) {
|
||||
<%=jobStructureCatcher.getUniqueName()%>,"<%=sourceNodeId%>","<%=sourceNodeComponent%>","<%=node.getUniqueName()%>","<%=node.getComponent().getName()%>","<%="REJECT".equals(con.getConnectorName()) ? "reject" : "output"%>")) {
|
||||
<%=jobStructureCatcher.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
}
|
||||
<%
|
||||
|
||||
@@ -145,9 +145,6 @@
|
||||
String cid = node.getUniqueName();
|
||||
boolean logstashCurrent = !cid.startsWith("tJobStructureCatcher") && !cid.startsWith("talend") && enableLogStash;
|
||||
|
||||
//about performance monitor, no way to support more than one job catcher component, also that is not necessary
|
||||
final String subprocessName4Catcher = logstashCurrent ? jobCatcherNodes.get(0).getDesignSubjobStartNode().getUniqueName() : null;
|
||||
|
||||
if ((codePart.equals(ECodePart.BEGIN))&&(stat || logstashCurrent)&&connSet.size()>0) {
|
||||
if(containsTPartitioner) {
|
||||
%>
|
||||
@@ -184,35 +181,13 @@
|
||||
if(!node.getComponent().useMerge()) {
|
||||
if(stat && logstashCurrent) {
|
||||
%>
|
||||
if(runStat.update(execStat,enableLogStash,iterateId,1,1
|
||||
<%for(IConnection con : connSet){
|
||||
INode source = con.getSource();
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
%>
|
||||
,"<%=con.getUniqueName()%>","<%=sourceNodeId%>","<%=sourceNodeLabel%>","<%=sourceNodeComponent%>","<%=node.getUniqueName()%>","<%=targetNodeLabel%>","<%=node.getComponent().getName()%>"
|
||||
<%}%>
|
||||
)) {
|
||||
<%=subprocessName4Catcher %>Process(globalMap);
|
||||
}
|
||||
runStat.updateStatAndLog(execStat,enableLogStash,iterateId,1,1<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
<%
|
||||
} else {
|
||||
if(stat) {
|
||||
%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection(iterateId,1,1
|
||||
<%for(IConnection con : connSet){
|
||||
%>
|
||||
,"<%=con.getUniqueName()%>"
|
||||
<%}%>
|
||||
);
|
||||
runStat.updateStatOnConnection(iterateId,1,1<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
@@ -220,24 +195,7 @@
|
||||
if(logstashCurrent) {
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
if(runStat.updateLog(iterateId,1,1
|
||||
<%for(IConnection con : connSet){
|
||||
INode source = con.getSource();
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
%>
|
||||
,"<%=con.getUniqueName()%>","<%=sourceNodeId%>","<%=sourceNodeLabel%>","<%=sourceNodeComponent%>","<%=node.getUniqueName()%>","<%=targetNodeLabel%>","<%=node.getComponent().getName()%>"
|
||||
<%}%>
|
||||
)) {
|
||||
<%=subprocessName4Catcher %>Process(globalMap);
|
||||
}
|
||||
runStat.log(iterateId,1,1<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
@@ -247,62 +205,18 @@
|
||||
if(connection.getUniqueName().equals((String)codeGenArgument.getIncomingName())){
|
||||
if(stat && logstashCurrent) {
|
||||
%>
|
||||
if(runStat.update(execStat,enableLogStash,iterateId,1,1
|
||||
<%for(IConnection con : connSet){
|
||||
INode source = con.getSource();
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
|
||||
if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){%>
|
||||
,"<%=con.getUniqueName()%>","<%=sourceNodeId%>","<%=sourceNodeLabel%>","<%=sourceNodeComponent%>","<%=node.getUniqueName()%>","<%=targetNodeLabel%>","<%=node.getComponent().getName()%>"
|
||||
<%}
|
||||
}%>
|
||||
)) {
|
||||
<%=subprocessName4Catcher %>Process(globalMap);
|
||||
}
|
||||
runStat.updateStatAndLog(execStat,enableLogStash,iterateId,1,1<%for(IConnection con : connSet){if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){%>,"<%=con.getUniqueName()%>"<%}}%>);
|
||||
<%
|
||||
} else {
|
||||
if(stat) {%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection(iterateId,1,1
|
||||
<%for(IConnection con : connSet){
|
||||
if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){%>
|
||||
,"<%=con.getUniqueName()%>"
|
||||
<%}
|
||||
}%>
|
||||
);
|
||||
runStat.updateStatOnConnection(iterateId,1,1<%for(IConnection con : connSet){if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){%>,"<%=con.getUniqueName()%>"<%}}%>);
|
||||
}
|
||||
<%}
|
||||
|
||||
if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
if(runStat.updateLog(iterateId,1,1
|
||||
<%for(IConnection con : connSet){
|
||||
INode source = con.getSource();
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
|
||||
if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){%>
|
||||
,"<%=con.getUniqueName()%>","<%=sourceNodeId%>","<%=sourceNodeLabel%>","<%=sourceNodeComponent%>","<%=node.getUniqueName()%>","<%=targetNodeLabel%>","<%=node.getComponent().getName()%>"
|
||||
<%}
|
||||
}%>
|
||||
)) {
|
||||
<%=subprocessName4Catcher %>Process(globalMap);
|
||||
}
|
||||
runStat.log(iterateId,1,1<%for(IConnection con : connSet){if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){%>,"<%=con.getUniqueName()%>"<%}}%>);
|
||||
}
|
||||
<%}
|
||||
}
|
||||
@@ -334,11 +248,9 @@
|
||||
|
||||
if(logstashCurrent) {
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
String label = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String nodeLabel = ((label==null || "__UNIQUE_NAME__".equals(label) || label.contains("\"")) ? node.getUniqueName() : label);
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
<%=jobStructureCatcher.getUniqueName() %>.addCM("<%=node.getUniqueName()%>", "<%=nodeLabel%>", "<%=node.getComponent().getName()%>");
|
||||
<%=jobStructureCatcher.getUniqueName() %>.addCM("<%=node.getUniqueName()%>", "<%=node.getComponent().getName()%>");
|
||||
<%=jobStructureCatcher.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
}
|
||||
<%
|
||||
|
||||
@@ -275,7 +275,7 @@
|
||||
}
|
||||
%>
|
||||
|
||||
protected PropertiesWithType context_param = new PropertiesWithType();
|
||||
private PropertiesWithType context_param = new PropertiesWithType();
|
||||
public java.util.Map<String, Object> parentContextMap = new java.util.HashMap<String, Object>();
|
||||
|
||||
public String status= "";
|
||||
@@ -464,7 +464,7 @@
|
||||
lastStr = "";
|
||||
}
|
||||
}
|
||||
enableLogStash = "true".equalsIgnoreCase(System.getProperty("audit.enabled"));
|
||||
enableLogStash = "true".equalsIgnoreCase(System.getProperty("monitoring"));
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(!"".equals(log4jLevel)){
|
||||
@@ -536,15 +536,15 @@
|
||||
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("host", "false");
|
||||
|
||||
System.getProperties().stringPropertyNames().stream()
|
||||
.filter(it -> it.startsWith("audit.logger."))
|
||||
.forEach(key -> properties_<%=jobCatcherNode.getUniqueName()%>.setProperty(key.substring("audit.logger.".length()), System.getProperty(key)));
|
||||
.filter(it -> it.startsWith("monitoring.audit.logger.properties."))
|
||||
.forEach(key -> properties_<%=jobCatcherNode.getUniqueName()%>.setProperty(key.substring("monitoring.audit.logger.properties.".length()), System.getProperty(key)));
|
||||
|
||||
<%if(isLog4j1Enabled) {%>
|
||||
org.apache.log4j.Logger.getLogger(properties_<%=jobCatcherNode.getUniqueName()%>.getProperty("root.logger")).setLevel(org.apache.log4j.Level.DEBUG);
|
||||
org.apache.log4j.Logger.getLogger("audit").setLevel(org.apache.log4j.Level.DEBUG);
|
||||
<%}%>
|
||||
|
||||
<%if(isLog4j2Enabled) {%>
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel(properties_<%=jobCatcherNode.getUniqueName()%>.getProperty("root.logger"), org.apache.logging.log4j.Level.DEBUG);
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel("audit", org.apache.logging.log4j.Level.DEBUG);
|
||||
<%}%>
|
||||
auditLogger_<%=jobCatcherNode.getUniqueName()%> = org.talend.job.audit.JobEventAuditLoggerFactory.createJobAuditLogger(properties_<%=jobCatcherNode.getUniqueName()%>);
|
||||
}
|
||||
@@ -926,12 +926,10 @@ this.globalResumeTicket = true;//to run tPreJob
|
||||
// run tHadoopConfManager component if there is it.
|
||||
if(!hadoopConfNodes.isEmpty()) {
|
||||
INode hadoopConfNode = hadoopConfNodes.get(0);
|
||||
if (hadoopConfNode.isStart()) {
|
||||
%>
|
||||
<%=createCallProcess(hadoopConfNode, className, false) %>
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
<%
|
||||
@@ -1345,11 +1343,13 @@ if (execStat) {
|
||||
} else {
|
||||
type = "ftp";
|
||||
}
|
||||
if ("sftp".equals(type)) { %>
|
||||
if ("ftp".equals(type)) { %>
|
||||
((com.enterprisedt.net.ftp.FTPClient) obj_conn).quit();
|
||||
<%} else if ("sftp".equals(type)) { %>
|
||||
((com.jcraft.jsch.ChannelSftp) obj_conn).quit();
|
||||
<%} else { %>
|
||||
((org.apache.commons.net.ftp.FTPClient) obj_conn).disconnect();
|
||||
<%}%>
|
||||
<%} else { %>
|
||||
((org.apache.commons.net.ftp.FTPSClient) obj_conn).disconnect();
|
||||
<%}%>
|
||||
}
|
||||
<% } %>
|
||||
} catch (java.lang.Exception e) {
|
||||
@@ -1439,24 +1439,11 @@ if (execStat) {
|
||||
private void closeGSConnections() {
|
||||
try {
|
||||
<% for (INode gsComponent : gsComponentsList) { %>
|
||||
|
||||
<%
|
||||
String authType = ElementParameterParser.getValue(gsComponent, "__AUTH_TYPE__");
|
||||
if (authType.equals("USER_ACCOUNT_HMAC")){
|
||||
%>
|
||||
|
||||
org.jets3t.service.impl.rest.httpclient.GoogleStorageService service_<%=gsComponent.getUniqueName()%> =
|
||||
(org.jets3t.service.impl.rest.httpclient.GoogleStorageService)globalMap.get("service_<%=gsComponent.getUniqueName()%>");
|
||||
if(service_<%=gsComponent.getUniqueName()%> !=null){
|
||||
service_<%=gsComponent.getUniqueName()%>.shutdown();
|
||||
}
|
||||
<%
|
||||
} else if (authType.equals("SERVICE_ACCOUNT")) {
|
||||
%>
|
||||
//Don't need to close storage when use google storage api
|
||||
<%
|
||||
}
|
||||
%>
|
||||
org.jets3t.service.impl.rest.httpclient.GoogleStorageService service_<%=gsComponent.getUniqueName()%> =
|
||||
(org.jets3t.service.impl.rest.httpclient.GoogleStorageService)globalMap.get("service_<%=gsComponent.getUniqueName()%>");
|
||||
if(service_<%=gsComponent.getUniqueName()%> !=null){
|
||||
service_<%=gsComponent.getUniqueName()%>.shutdown();
|
||||
}
|
||||
|
||||
<% } %>
|
||||
} catch (java.lang.Exception e) {
|
||||
@@ -1608,7 +1595,7 @@ if (execStat) {
|
||||
}
|
||||
} else if (arg.startsWith("--log4jLevel=")) {
|
||||
log4jLevel = arg.substring(13);
|
||||
} else if (arg.startsWith("--audit.enabled") && arg.contains("=")) {//for trunjob call
|
||||
} else if (arg.startsWith("--monitoring") && arg.contains("=")) {//for trunjob call
|
||||
final int equal = arg.indexOf('=');
|
||||
final String key = arg.substring("--".length(), equal);
|
||||
System.setProperty(key, arg.substring(equal + 1));
|
||||
|
||||
@@ -62,6 +62,8 @@ class IndexedRecordToRowStructGenerator {
|
||||
%>
|
||||
boolean <%=codeVarIsDynamicInitialized%> = false;
|
||||
routines.system.Dynamic <%=codeVarDynamic%> = new routines.system.Dynamic();
|
||||
//Workaround for TDI-44051, TcimpV0 do not need DbmsID
|
||||
<%=codeVarDynamic%>.setDbmsId("<%=cid%>");
|
||||
<%
|
||||
}
|
||||
|
||||
@@ -115,14 +117,11 @@ class IndexedRecordToRowStructGenerator {
|
||||
for (org.apache.avro.Schema.Field dynamicField_<%=cid%> : dynSchema_<%=cid%>.getFields()){
|
||||
routines.system.DynamicMetadata dynamicMetadata_<%=cid%> = new routines.system.DynamicMetadata();
|
||||
org.apache.avro.Schema dynamicFieldSchema_<%=cid%> = dynamicField_<%=cid%>.schema();
|
||||
String dbName = dynamicField_<%=cid%>.getProp("talend.field.dbColumnName");
|
||||
// set name
|
||||
if("true".equals(dynamicField_<%=cid%>.getProp("ENABLE_SPECIAL_TABLENAME"))){
|
||||
dynamicMetadata_<%=cid%>.setName(dynamicField_<%=cid%>.getProp("talend.field.dbColumnName"));
|
||||
}else{
|
||||
dynamicMetadata_<%=cid%>.setName(dynamicField_<%=cid%>.name());
|
||||
}
|
||||
dynamicMetadata_<%=cid%>.setName(dynamicField_<%=cid%>.name());
|
||||
// set db name
|
||||
dynamicMetadata_<%=cid%>.setDbName(dynamicField_<%=cid%>.name());
|
||||
dynamicMetadata_<%=cid%>.setDbName(dbName==null?dynamicField_<%=cid%>.name():dbName);
|
||||
// set nullable
|
||||
if (org.talend.daikon.avro.AvroUtils.isNullable(dynamicFieldSchema_<%=cid%>)) {
|
||||
dynamicMetadata_<%=cid%>.setNullable(true);
|
||||
@@ -170,14 +169,6 @@ class IndexedRecordToRowStructGenerator {
|
||||
if (columnName.equals(dynamicColName)) {
|
||||
%>
|
||||
java.util.Map<String, Object> dynamicValue_<%=cid%> = (java.util.Map<String, Object>) <%=codeVarSchemaEnforcer%>.get(<%=i%>);
|
||||
org.apache.avro.Schema dynSchema_<%=cid%> = ((org.talend.codegen.enforcer.OutgoingDynamicSchemaEnforcer) <%=codeVarSchemaEnforcer%>).getDynamicFieldsSchema();
|
||||
for (org.apache.avro.Schema.Field dynamicField_<%=cid%> : dynSchema_<%=cid%>.getFields()){
|
||||
String name = dynamicField_<%=cid%>.name();
|
||||
if("true".equals(dynamicField_<%=cid%>.getProp("ENABLE_SPECIAL_TABLENAME"))){
|
||||
dynamicValue_<%=cid%>.put(dynamicField_<%=cid%>.getProp("talend.field.dbColumnName"), dynamicValue_<%=cid%>.get(name));
|
||||
dynamicValue_<%=cid%>.remove(name);
|
||||
}
|
||||
}
|
||||
for (java.util.Map.Entry<String, Object> dynamicValueEntry_<%=cid%> : dynamicValue_<%=cid%>.entrySet()) {
|
||||
<%=codeVarDynamic%>.setColumnValue(<%=codeVarDynamic%>.getIndex(dynamicValueEntry_<%=cid%>.getKey()), dynamicValueEntry_<%=cid%>.getValue());
|
||||
}
|
||||
|
||||
@@ -390,14 +390,7 @@ public <%=JavaTypesManager.getTypeToGenerate(ctxParam.getType(),true)%> get<%=Ch
|
||||
}
|
||||
}
|
||||
boolean enableLogStash = jobCatcherNode != null;
|
||||
if (enableLogStash) {
|
||||
%>
|
||||
private final JobStructureCatcherUtils <%=jobCatcherNode.getUniqueName() %> = new JobStructureCatcherUtils(jobName, "<%=process.getId() %>", "<%=process.getVersion() %>");
|
||||
private org.talend.job.audit.JobAuditLogger auditLogger_<%=jobCatcherNode.getUniqueName()%> = null;
|
||||
|
||||
private RunStat runStat = new RunStat(<%=jobCatcherNode.getUniqueName() %>);
|
||||
<%
|
||||
} else if(stats) {
|
||||
if (stats || enableLogStash) {
|
||||
%>
|
||||
private RunStat runStat = new RunStat();
|
||||
<%
|
||||
@@ -438,6 +431,13 @@ private RunTrace runTrace = new RunTrace();
|
||||
<%
|
||||
}
|
||||
|
||||
if(enableLogStash) {
|
||||
%>
|
||||
JobStructureCatcherUtils <%=jobCatcherNode.getUniqueName() %> = new JobStructureCatcherUtils(jobName, "<%=process.getId() %>", "<%=process.getVersion() %>");
|
||||
org.talend.job.audit.JobAuditLogger auditLogger_<%=jobCatcherNode.getUniqueName()%> = null;
|
||||
<%
|
||||
}
|
||||
|
||||
for (INode metterCatcher : process.getNodesOfType("tFlowMeterCatcher")) {
|
||||
%>
|
||||
MetterCatcherUtils <%=metterCatcher.getUniqueName() %> = new MetterCatcherUtils("<%=process.getId() %>", "<%=process.getVersion() %>");
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.codegen</artifactId>
|
||||
|
||||
@@ -66,7 +66,6 @@ import org.talend.designer.codegen.model.CodeGeneratorEmittersPoolFactory;
|
||||
import org.talend.designer.codegen.model.CodeGeneratorInternalTemplatesFactoryProvider;
|
||||
import org.talend.designer.codegen.proxy.JetProxy;
|
||||
import org.talend.designer.core.generic.model.Component;
|
||||
import org.talend.designer.runprocess.ProcessorUtilities;
|
||||
|
||||
/**
|
||||
* CodeGenerator.
|
||||
@@ -354,12 +353,8 @@ public class CodeGenerator implements ICodeGenerator {
|
||||
.getDefault().getService(IResourcesDependenciesService.class);
|
||||
String resourcePathForContext = null;
|
||||
if (process instanceof IProcess2) {
|
||||
String value = contextPar.getValue();
|
||||
// for runtime
|
||||
if (!ProcessorUtilities.isExportConfig()) {
|
||||
contextPar.setType(JavaTypesManager.FILE.getId());
|
||||
}
|
||||
resourcePathForContext = resourceService.getResourcePathForContext(process, value);
|
||||
resourcePathForContext = resourceService.getResourcePathForContext(process,
|
||||
contextPar.getValue());
|
||||
} else {
|
||||
// for PreviewFileInputContentDataProcess run
|
||||
resourcePathForContext = resourceService.getResourceItemFilePath(contextPar.getValue());
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.components.exchange</artifactId>
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>google-storage-utils</artifactId>
|
||||
<version>1.1.0</version>
|
||||
<version>1.0.0</version>
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
@@ -46,12 +46,6 @@
|
||||
<artifactId>jets3t</artifactId>
|
||||
<version>0.9.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter-api</artifactId>
|
||||
<version>5.6.2</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
@@ -70,8 +64,8 @@
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>8</source>
|
||||
<target>8</target>
|
||||
<source>1.6</source>
|
||||
<target>1.6</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
|
||||
@@ -3,13 +3,10 @@ package org.talend.gs.util;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.jets3t.service.model.GSObject;
|
||||
@@ -47,50 +44,6 @@ public class GSObjectUtil {
|
||||
return objects;
|
||||
}
|
||||
|
||||
public Map<String, File> generateFileMap(File rootFile, String directory, Map<String, String> filter){
|
||||
Map<String, File> allFiles = generateFileMap(rootFile, directory);
|
||||
Map<String, File> resultMap = new HashMap<>();
|
||||
for (Map.Entry<String, String> entry : filter.entrySet()) {
|
||||
//key is path to file on local machine
|
||||
//value is new path in GS
|
||||
File currentFile = new File(entry.getKey());
|
||||
if (allFiles.containsValue(currentFile)){
|
||||
resultMap.put(entry.getValue(), currentFile);
|
||||
}
|
||||
}
|
||||
return resultMap;
|
||||
}
|
||||
|
||||
public Map<String, File> generateFileMap(File rootFile, String directory){
|
||||
if (rootFile == null){
|
||||
throw new IllegalArgumentException("File can't be null");
|
||||
}
|
||||
if (directory == null){
|
||||
throw new IllegalArgumentException("Directory name can't be null");
|
||||
}
|
||||
|
||||
if (!directory.isEmpty() && !"/".equals(directory.substring(directory.length()-1))){
|
||||
directory = directory + "/";
|
||||
}
|
||||
|
||||
Map<String,File> fileMap = new HashMap<>();
|
||||
|
||||
if (!rootFile.isDirectory()){
|
||||
fileMap.put(directory + rootFile.getName(), rootFile);
|
||||
return fileMap;
|
||||
}
|
||||
|
||||
String pathToDirectory = rootFile.getAbsolutePath();
|
||||
List<File> allFiles = listAllFiles(pathToDirectory);
|
||||
|
||||
for (File currentFile : allFiles){
|
||||
if (!currentFile.isDirectory()){
|
||||
fileMap.put(combineTwoNames(pathToDirectory, directory, currentFile), currentFile);
|
||||
}
|
||||
}
|
||||
return fileMap;
|
||||
}
|
||||
|
||||
public java.util.Map<String, String> genFileFilterList(java.util.List<java.util.Map<String, String>> list,
|
||||
String localdir, String remotedir) {
|
||||
if (remotedir != null && !"".equals(remotedir)) {
|
||||
@@ -167,23 +120,4 @@ public class GSObjectUtil {
|
||||
InputStream mimetypesFile = this.getClass().getResourceAsStream("/resource/mime.types");
|
||||
Mimetypes.getInstance().loadAndReplaceMimetypes(mimetypesFile);
|
||||
}
|
||||
|
||||
private List<File> listAllFiles(String directoryName) {
|
||||
File directory = new File(directoryName);
|
||||
List<File> resultList = new ArrayList<>();
|
||||
File[] fList = directory.listFiles();
|
||||
resultList.addAll(Arrays.asList(fList));
|
||||
for (File file : fList) {
|
||||
if (file.isDirectory()) {
|
||||
resultList.addAll(listAllFiles(file.getAbsolutePath()));
|
||||
}
|
||||
}
|
||||
return resultList;
|
||||
}
|
||||
|
||||
private String combineTwoNames(String coreDir, String targetDir, File file){
|
||||
String filePath = file.getAbsolutePath();
|
||||
String relativePath = targetDir + filePath.replace(coreDir + FileSystems.getDefault().getSeparator(), "");
|
||||
return relativePath.replaceAll("\\\\","/");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
package util;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.talend.gs.util.GSObjectUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class GSObjectUtilTest {
|
||||
|
||||
@Test
|
||||
void checkSizeOfFilesInFolderTest() {
|
||||
Path pathToDirectory = Paths.get("src", "test", "resources", "dirWithThreeFiles");
|
||||
Path pathToFile1 = Paths.get("src", "test", "resources", "dirWithThreeFiles", "1.csv");
|
||||
Path pathToFile2 = Paths.get("src", "test", "resources", "dirWithThreeFiles", "2.csv");
|
||||
Path pathToFile3 = Paths.get("src", "test", "resources", "dirWithThreeFiles", "3.csv");
|
||||
Path pathToFile4 = Paths.get("src", "test", "resources", "dirWithThreeFiles", "inner", "1.csv");
|
||||
|
||||
String expectedRelativePathToFile1 = "/1.csv";
|
||||
String expectedRelativePathToFile2 = "/2.csv";
|
||||
String expectedRelativePathToFile3 = "/3.csv";
|
||||
String expectedRelativePathToFile4 = "/inner/1.csv";
|
||||
|
||||
File rootFile = new File(pathToDirectory.toUri());
|
||||
GSObjectUtil gsObjectUtil = new GSObjectUtil();
|
||||
Map<String, File> files = gsObjectUtil.generateFileMap(rootFile, "/");
|
||||
assertEquals(4, files.size());
|
||||
|
||||
File expectedFile1 = new File(pathToFile1.toUri());
|
||||
File expectedFile2 = new File(pathToFile2.toUri());
|
||||
File expectedFile3 = new File(pathToFile3.toUri());
|
||||
File expectedFile4 = new File(pathToFile4.toUri());
|
||||
|
||||
assertTrue(files.containsKey(expectedRelativePathToFile1));
|
||||
assertTrue(files.containsKey(expectedRelativePathToFile2));
|
||||
assertTrue(files.containsKey(expectedRelativePathToFile3));
|
||||
assertTrue(files.containsKey(expectedRelativePathToFile4));
|
||||
|
||||
assertEquals(expectedFile1.getAbsoluteFile(), files.get(expectedRelativePathToFile1).getAbsoluteFile());
|
||||
assertEquals(expectedFile2.getAbsoluteFile(), files.get(expectedRelativePathToFile2).getAbsoluteFile());
|
||||
assertEquals(expectedFile3.getAbsoluteFile(), files.get(expectedRelativePathToFile3).getAbsoluteFile());
|
||||
assertEquals(expectedFile4.getAbsoluteFile(), files.get(expectedRelativePathToFile4).getAbsoluteFile());
|
||||
}
|
||||
|
||||
@Test
|
||||
void checkSizeOfFilesInFolder2Test() {
|
||||
Path pathToDirectory = Paths.get("src", "test", "resources", "dirWithThreeFiles");
|
||||
Path pathToFile1 = Paths.get("src", "test", "resources", "dirWithThreeFiles", "1.csv");
|
||||
Path pathToFile2 = Paths.get("src", "test", "resources", "dirWithThreeFiles", "2.csv");
|
||||
Path pathToFile3 = Paths.get("src", "test", "resources", "dirWithThreeFiles", "3.csv");
|
||||
Path pathToFile4 = Paths.get("src", "test", "resources", "dirWithThreeFiles", "inner", "1.csv");
|
||||
|
||||
String expectedRelativePathToFile1 = "1.csv";
|
||||
String expectedRelativePathToFile2 = "2.csv";
|
||||
String expectedRelativePathToFile3 = "3.csv";
|
||||
String expectedRelativePathToFile4 = "inner/1.csv";
|
||||
|
||||
File rootFile = new File(pathToDirectory.toUri());
|
||||
GSObjectUtil gsObjectUtil = new GSObjectUtil();
|
||||
Map<String, File> files = gsObjectUtil.generateFileMap(rootFile, "");
|
||||
assertEquals(4, files.size());
|
||||
|
||||
File expectedFile1 = new File(pathToFile1.toUri());
|
||||
File expectedFile2 = new File(pathToFile2.toUri());
|
||||
File expectedFile3 = new File(pathToFile3.toUri());
|
||||
File expectedFile4 = new File(pathToFile4.toUri());
|
||||
|
||||
assertTrue(files.containsKey(expectedRelativePathToFile1));
|
||||
assertTrue(files.containsKey(expectedRelativePathToFile2));
|
||||
assertTrue(files.containsKey(expectedRelativePathToFile3));
|
||||
assertTrue(files.containsKey(expectedRelativePathToFile4));
|
||||
|
||||
assertEquals(expectedFile1.getAbsoluteFile(), files.get(expectedRelativePathToFile1).getAbsoluteFile());
|
||||
assertEquals(expectedFile2.getAbsoluteFile(), files.get(expectedRelativePathToFile2).getAbsoluteFile());
|
||||
assertEquals(expectedFile3.getAbsoluteFile(), files.get(expectedRelativePathToFile3).getAbsoluteFile());
|
||||
assertEquals(expectedFile4.getAbsoluteFile(), files.get(expectedRelativePathToFile4).getAbsoluteFile());
|
||||
}
|
||||
|
||||
@Test
|
||||
void oneFileWithCustomPathTest(){
|
||||
Path pathToFile = Paths.get("src","test", "resources", "empty.csv");
|
||||
String pathToFolder = "/folder/folder";
|
||||
File expectedFile = new File(pathToFile.toUri());
|
||||
GSObjectUtil gsObjectUtil = new GSObjectUtil();
|
||||
|
||||
Map<String, File> files = gsObjectUtil.generateFileMap(expectedFile, pathToFolder);
|
||||
|
||||
String expectedRelativePathToFile = "/folder/folder/empty.csv";
|
||||
|
||||
assertEquals(1, files.size());
|
||||
assertTrue(files.containsKey(expectedRelativePathToFile));
|
||||
assertEquals(expectedFile.getAbsoluteFile(), files.get(expectedRelativePathToFile).getAbsoluteFile());
|
||||
}
|
||||
|
||||
@Test
|
||||
void oneFileWithOtherCustomPathTest(){
|
||||
Path pathToFile = Paths.get("src","test", "resources", "empty.csv");
|
||||
String pathToFolder = "folder/folder";
|
||||
File expectedFile = new File(pathToFile.toUri());
|
||||
GSObjectUtil gsObjectUtil = new GSObjectUtil();
|
||||
|
||||
Map<String, File> files = gsObjectUtil.generateFileMap(expectedFile, pathToFolder);
|
||||
|
||||
String expectedRelativePathToFile = "folder/folder/empty.csv";
|
||||
|
||||
assertEquals(1, files.size());
|
||||
assertTrue(files.containsKey(expectedRelativePathToFile));
|
||||
assertEquals(expectedFile.getAbsoluteFile(), files.get(expectedRelativePathToFile).getAbsoluteFile());
|
||||
}
|
||||
|
||||
@Test
|
||||
void folderWithOtherCustomPathTest(){
|
||||
Path pathToFolder = Paths.get("src", "test", "resources", "dirWithThreeFiles", "inner");
|
||||
String pathToGSFolder = "folder";
|
||||
File folder = new File(pathToFolder.toUri());
|
||||
GSObjectUtil gsObjectUtil = new GSObjectUtil();
|
||||
|
||||
Map<String, File> files = gsObjectUtil.generateFileMap(folder, pathToGSFolder);
|
||||
|
||||
String expectedRelativePathToFile = "folder/1.csv";
|
||||
File expectedFile = new File(pathToFolder + "/1.csv");
|
||||
|
||||
assertEquals(1, files.size());
|
||||
assertTrue(files.containsKey(expectedRelativePathToFile));
|
||||
assertEquals(expectedFile.getAbsoluteFile(), files.get(expectedRelativePathToFile).getAbsoluteFile());
|
||||
}
|
||||
}
|
||||
|
|
|
|
|
|
|
|
|
|
@@ -170,7 +170,7 @@
|
||||
<dependency>
|
||||
<groupId>commons-beanutils</groupId>
|
||||
<artifactId>commons-beanutils</artifactId>
|
||||
<version>1.9.4</version>
|
||||
<version>1.8.0</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
@@ -310,7 +310,7 @@
|
||||
<link>http://java.sun.com/j2se/1.5.0/docs/api/</link>
|
||||
<link>http://www.junit.org/junit/javadoc/</link>
|
||||
<link>
|
||||
http://jakarta.apache.org/commons/beanutils/commons-beanutils-1.9.4/docs/api/
|
||||
http://jakarta.apache.org/commons/beanutils/commons-beanutils-1.7.0/docs/api/
|
||||
</link>
|
||||
<link>
|
||||
http://jakarta.apache.org/commons/lang/api-2.2/
|
||||
|
||||
@@ -51,8 +51,7 @@ public abstract class AbstractJSONObjectStaticBuildersTestCase extends TestCase
|
||||
jsonConfig.setIgnoreDefaultExcludes( true );
|
||||
JSONObject json = JSONObject.fromObject( getSource(), jsonConfig );
|
||||
assertJSONObject( json, getProperties() );
|
||||
//see : https://github.com/apache/commons-beanutils/compare/BEANUTILS_1_9_3...commons-beanutils-1.9.4#diff-dbeaa70d3a09169fb5a1e42e5b319037R34-R43
|
||||
//assertTrue( json.has( "class" ) );
|
||||
assertTrue( json.has( "class" ) );
|
||||
assertTrue( !json.has( "pexcluded" ) );
|
||||
}
|
||||
|
||||
|
||||
@@ -199,8 +199,7 @@ public class TestJSONObject extends TestCase {
|
||||
jsonConfig.setIgnoreDefaultExcludes( true );
|
||||
jsonObject.element( "bean", new ObjectBean(), jsonConfig );
|
||||
JSONObject actual = jsonObject.getJSONObject( "bean" );
|
||||
//see : https://github.com/apache/commons-beanutils/compare/BEANUTILS_1_9_3...commons-beanutils-1.9.4#diff-dbeaa70d3a09169fb5a1e42e5b319037R34-R43
|
||||
//Assertions.assertTrue( actual.has( "class" ) );
|
||||
Assertions.assertTrue( actual.has( "class" ) );
|
||||
Assertions.assertTrue( !actual.has( "pexcluded" ) );
|
||||
}
|
||||
|
||||
@@ -259,8 +258,7 @@ public class TestJSONObject extends TestCase {
|
||||
jsonObject.element( "list", list, jsonConfig );
|
||||
JSONObject actual = jsonObject.getJSONArray( "list" )
|
||||
.getJSONObject( 0 );
|
||||
//see : https://github.com/apache/commons-beanutils/compare/BEANUTILS_1_9_3...commons-beanutils-1.9.4#diff-dbeaa70d3a09169fb5a1e42e5b319037R34-R43
|
||||
//Assertions.assertTrue( actual.has( "class" ) );
|
||||
Assertions.assertTrue( actual.has( "class" ) );
|
||||
Assertions.assertTrue( !actual.has( "pexcluded" ) );
|
||||
}
|
||||
|
||||
|
||||
@@ -1,64 +1,26 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<groupId>org.talend</groupId>
|
||||
<artifactId>mdm_transaction</artifactId>
|
||||
<name>mdm_transaction</name>
|
||||
<version>1.0.1</version>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
</snapshotRepository>
|
||||
<repository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<version>1.7</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-httpclient</groupId>
|
||||
<artifactId>commons-httpclient</artifactId>
|
||||
<version>3.0</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<version>1.7</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
<version>1.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-httpclient</groupId>
|
||||
<artifactId>commons-httpclient</artifactId>
|
||||
<version>3.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
@@ -87,4 +49,5 @@
|
||||
</testResource>
|
||||
</testResources>
|
||||
</build>
|
||||
<version>1</version>
|
||||
</project>
|
||||
@@ -1,18 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>job-audit</artifactId>
|
||||
<version>1.2</version>
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>job-audit</artifactId>
|
||||
<version>1.1</version>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<daikon.audit.version>1.16.0</daikon.audit.version>
|
||||
</properties>
|
||||
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>talend_nexus</id>
|
||||
@@ -24,7 +23,7 @@
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
@@ -47,54 +46,47 @@
|
||||
</releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.talend.daikon</groupId>
|
||||
<artifactId>audit-common</artifactId>
|
||||
<version>1.8.0</version>
|
||||
</dependency>
|
||||
|
||||
<!--
|
||||
<dependency>
|
||||
<groupId>org.talend.daikon</groupId>
|
||||
<artifactId>audit-common</artifactId>
|
||||
<version>${daikon.audit.version}</version>
|
||||
<groupId>org.talend.daikon</groupId>
|
||||
<artifactId>audit-log4j1</artifactId>
|
||||
<version>1.8.0</version>
|
||||
</dependency>
|
||||
|
||||
<!-- <dependency> <groupId>org.talend.daikon</groupId> <artifactId>audit-log4j1</artifactId>
|
||||
<version>${daikon.audit.version}</version> </dependency> <dependency> <groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId> <version>1.2.17</version> <scope>test</scope>
|
||||
</dependency> -->
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.talend.daikon</groupId>
|
||||
<artifactId>audit-log4j2</artifactId>
|
||||
<version>${daikon.audit.version}</version>
|
||||
</dependency>
|
||||
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
-->
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-api</artifactId>
|
||||
<version>2.12.1</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
<version>2.12.1</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.5</version>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
<encoding>UTF-8</encoding>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
</build>
|
||||
<groupId>org.talend.daikon</groupId>
|
||||
<artifactId>audit-log4j2</artifactId>
|
||||
<version>1.8.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-api</artifactId>
|
||||
<version>2.12.1</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
<version>2.12.1</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -6,23 +6,20 @@ import org.talend.logging.audit.EventAuditLogger;
|
||||
import org.talend.logging.audit.LogLevel;
|
||||
|
||||
public interface JobAuditLogger extends EventAuditLogger {
|
||||
|
||||
@AuditEvent(category = "job", message = "Job start : job_name:{jobName}, job_version:{jobVersion}, job_id:{jobId}, timestamp:{timestamp}", level = LogLevel.INFO)
|
||||
void jobstart(Context context);
|
||||
|
||||
@AuditEvent(category = "job", message = "Job stop : job_name:{jobName}, job_version:{jobVersion}, job_id:{jobId}, timestamp:{timestamp}, status:{status}, duration:{duration}", level = LogLevel.INFO)
|
||||
void jobstop(Context context);
|
||||
|
||||
@AuditEvent(category = "connector", message = "Component run : job_name:{jobName}, job_version:{jobVersion}, job_id:{jobId}, connector_type:{connectorType}, connector_id:{connectorId}", level = LogLevel.INFO)
|
||||
void runcomponent(Context context);
|
||||
|
||||
@AuditEvent(category = "jobstart", message = "Job start : job_name:{job_name}, job_version:{job_version}, job_id:{job_id}, timestamp:{timestamp}", level = LogLevel.INFO)
|
||||
void jobstart(Context context);
|
||||
|
||||
@AuditEvent(category = "jobstop", message = "Job stop : job_name:{job_name}, job_version:{job_version}, job_id:{job_id}, timestamp:{timestamp}, status:{status}, duration:{duration}", level = LogLevel.INFO)
|
||||
void jobstop(Context context);
|
||||
|
||||
@AuditEvent(category = "runcomponent", message = "Component run : job_name:{job_name}, job_version:{job_version}, job_id:{job_id}, connector_type:{connector_type}, connector_id:{connector_id}, connector_label:{connector_label}", level = LogLevel.INFO)
|
||||
void runcomponent(Context context);
|
||||
|
||||
@AuditEvent(category = "flowoutput", message = "Component {connector_type} {connector_label} {connection_type} {rows} rows in {duration} with {connection_name} line", level = LogLevel.INFO)
|
||||
void flowOutput(Context context);
|
||||
|
||||
@AuditEvent(category = "flowinput", message = "Component {connector_type} {connector_label} received {rows} rows in {duration} with {connection_name} line", level = LogLevel.INFO)
|
||||
void flowInput(Context context);
|
||||
|
||||
@AuditEvent(category = "flowexecution", message = "connection : {connection_name}, row : {rows}, cost : {duration}", level = LogLevel.INFO)
|
||||
void flowExecution(Context context);
|
||||
@AuditEvent(category = "connection", message = "Component {connectorType} {connectionType} {rows} rows in {duration} with {connectionName} line", level = LogLevel.INFO)
|
||||
void flowOutput(Context context);
|
||||
|
||||
@AuditEvent(category = "connection", message = "Component {connectorType} received {rows} rows in {duration} with {connectionName} line", level = LogLevel.INFO)
|
||||
void flowInput(Context context);
|
||||
|
||||
}
|
||||
|
||||
@@ -5,160 +5,86 @@ import org.talend.logging.audit.ContextBuilder;
|
||||
|
||||
public class JobContextBuilder {
|
||||
|
||||
private final ContextBuilder builder;
|
||||
private final ContextBuilder builder;
|
||||
|
||||
public JobContextBuilder(ContextBuilder builder) {
|
||||
this.builder = builder;
|
||||
}
|
||||
public JobContextBuilder(ContextBuilder builder) {
|
||||
this.builder = builder;
|
||||
}
|
||||
|
||||
public static JobContextBuilder create() {
|
||||
return new JobContextBuilder(ContextBuilder.create());
|
||||
}
|
||||
public static JobContextBuilder create() {
|
||||
return new JobContextBuilder(ContextBuilder.create());
|
||||
}
|
||||
|
||||
public JobContextBuilder jobName(String job_name) {
|
||||
builder.with("job_name", job_name);
|
||||
return this;
|
||||
}
|
||||
public JobContextBuilder jobName(String jobName) {
|
||||
builder.with("jobName", jobName);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder jobVersion(String jobVersion) {
|
||||
builder.with("jobVersion", jobVersion);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder jobId(String jobId) {
|
||||
builder.with("jobId", jobId);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* component type like "tXMLMap"
|
||||
* @param connectorType
|
||||
* @return self
|
||||
*/
|
||||
public JobContextBuilder connectorType(String connectorType) {
|
||||
builder.with("connectorType", connectorType);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* component type like "tXMLMap_1"
|
||||
* @param connectorType
|
||||
* @return self
|
||||
*/
|
||||
public JobContextBuilder connectorId(String connectorId) {
|
||||
builder.with("connectorId", connectorId);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder rows(long rowCount) {
|
||||
builder.with("rows", String.valueOf(rowCount));
|
||||
return this;
|
||||
}
|
||||
|
||||
//output or reject
|
||||
public JobContextBuilder connectionType(String connectionType) {
|
||||
builder.with("connectionType", connectionType);
|
||||
return this;
|
||||
}
|
||||
|
||||
//like "row1"
|
||||
public JobContextBuilder connectionName(String connectionName) {
|
||||
builder.with("connectionName", connectionName);
|
||||
return this;
|
||||
}
|
||||
|
||||
//100s
|
||||
public JobContextBuilder duration(String duration) {
|
||||
builder.with("duration", duration);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder timestamp(String timestamp) {
|
||||
builder.with("timestamp", timestamp);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder status(String status) {
|
||||
builder.with("status", status);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder jobVersion(String job_version) {
|
||||
builder.with("job_version", job_version);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder jobId(String job_id) {
|
||||
builder.with("job_id", job_id);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* component label, default is unique name like "tXMLMap_1", but user can adjust
|
||||
* it in studio to any value
|
||||
*
|
||||
* @param connectorType
|
||||
* @return self
|
||||
*/
|
||||
public JobContextBuilder connectorLabel(String connector_label) {
|
||||
builder.with("connector_label", connector_label);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* component type like "tXMLMap"
|
||||
*
|
||||
* @param connectorType
|
||||
* @return self
|
||||
*/
|
||||
public JobContextBuilder connectorType(String connector_type) {
|
||||
builder.with("connector_type", connector_type);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* component unique name like "tXMLMap_1"
|
||||
*
|
||||
* @param connectorType
|
||||
* @return self
|
||||
*/
|
||||
public JobContextBuilder connectorId(String connector_id) {
|
||||
builder.with("connector_id", connector_id);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder rows(long rows) {
|
||||
builder.with("rows", String.valueOf(rows));
|
||||
return this;
|
||||
}
|
||||
|
||||
// output or reject
|
||||
public JobContextBuilder connectionType(String connection_type) {
|
||||
builder.with("connection_type", connection_type);
|
||||
return this;
|
||||
}
|
||||
|
||||
// like "row1"
|
||||
public JobContextBuilder connectionName(String connection_name) {
|
||||
builder.with("connection_name", connection_name);
|
||||
return this;
|
||||
}
|
||||
|
||||
// 100s
|
||||
public JobContextBuilder duration(String duration) {
|
||||
builder.with("duration", duration);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder timestamp(String timestamp) {
|
||||
builder.with("timestamp", timestamp);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder status(String status) {
|
||||
builder.with("status", status);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Context build() {
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* source connector id
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public JobContextBuilder sourceId(String source_id) {
|
||||
builder.with("source_id", source_id);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* source connector label
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public JobContextBuilder sourceLabel(String source_label) {
|
||||
builder.with("source_label", source_label);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* source connector name
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public JobContextBuilder sourceConnectorType(String source_connector_type) {
|
||||
builder.with("source_connector_type", source_connector_type);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* target connector id
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public JobContextBuilder targetId(String target_id) {
|
||||
builder.with("target_id", target_id);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* target connector label
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public JobContextBuilder targetLabel(String target_label) {
|
||||
builder.with("target_label", target_label);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* target connector name
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public JobContextBuilder targetConnectorType(String target_connector_type) {
|
||||
builder.with("target_connector_type", target_connector_type);
|
||||
return this;
|
||||
}
|
||||
public Context build() {
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -5,47 +5,45 @@ import java.util.Properties;
|
||||
|
||||
import org.talend.logging.audit.AuditLoggerFactory;
|
||||
import org.talend.logging.audit.EventAuditLogger;
|
||||
import org.talend.logging.audit.impl.*;
|
||||
import org.talend.logging.audit.impl.AbstractBackend;
|
||||
import org.talend.logging.audit.impl.AuditConfiguration;
|
||||
import org.talend.logging.audit.impl.AuditConfigurationMap;
|
||||
import org.talend.logging.audit.impl.AuditLoggerBase;
|
||||
import org.talend.logging.audit.impl.DefaultAuditLoggerBase;
|
||||
import org.talend.logging.audit.impl.ProxyEventAuditLogger;
|
||||
|
||||
public class JobEventAuditLoggerFactory {
|
||||
|
||||
public static JobAuditLogger createJobAuditLogger(final Properties properties) {
|
||||
final AuditConfigurationMap config = AuditConfiguration.loadFromProperties(properties);
|
||||
|
||||
final DefaultAuditLoggerBase loggerBase;
|
||||
AbstractBackend logger = null;
|
||||
|
||||
//load log4j2 implement firstly
|
||||
String loggerClass = "org.talend.logging.audit.log4j2.Log4j2Backend";
|
||||
try {
|
||||
final Class<?> clz = Class.forName(loggerClass);
|
||||
logger = (AbstractBackend) clz.getConstructor(AuditConfigurationMap.class).newInstance(config);
|
||||
} catch (ReflectiveOperationException e) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
final Backends backend = AuditConfiguration.BACKEND.getValue(config, Backends.class);
|
||||
if (backend == Backends.CUSTOM) {
|
||||
loggerBase = new DefaultAuditLoggerBase(config);
|
||||
} else {
|
||||
AbstractBackend logger = null;
|
||||
|
||||
// load log4j2 implement firstly
|
||||
String loggerClass = "org.talend.logging.audit.log4j2.Log4j2Backend";
|
||||
//load log4j1 implement if not found log4j2
|
||||
if (logger == null) {
|
||||
loggerClass = "org.talend.logging.audit.log4j1.Log4j1Backend";
|
||||
try {
|
||||
final Class<?> clz = Class.forName(loggerClass);
|
||||
logger = (AbstractBackend) clz.getConstructor(AuditConfigurationMap.class).newInstance(config);
|
||||
} catch (ReflectiveOperationException e) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
// load log4j1 implement if not found log4j2
|
||||
if (logger == null) {
|
||||
loggerClass = "org.talend.logging.audit.log4j1.Log4j1Backend";
|
||||
try {
|
||||
final Class<?> clz = Class.forName(loggerClass);
|
||||
logger = (AbstractBackend) clz.getConstructor(AuditConfigurationMap.class).newInstance(config);
|
||||
} catch (ReflectiveOperationException e) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
|
||||
if (logger == null) {
|
||||
throw new RuntimeException("Unable to load backend : " + loggerClass);
|
||||
}
|
||||
|
||||
loggerBase = new DefaultAuditLoggerBase(logger, config);
|
||||
}
|
||||
|
||||
if(logger == null) {
|
||||
throw new RuntimeException("Unable to load backend : " + loggerClass);
|
||||
}
|
||||
|
||||
final DefaultAuditLoggerBase loggerBase = new DefaultAuditLoggerBase(logger, config);
|
||||
|
||||
JobAuditLogger result = getEventAuditLogger(JobAuditLogger.class, loggerBase);
|
||||
return result;
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>talend-proxy</artifactId>
|
||||
<version>1.0.1</version>
|
||||
<version>1.0.0</version>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
@@ -46,9 +46,9 @@
|
||||
<version>3.9</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.25</version>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
@@ -1,20 +1,22 @@
|
||||
package org.talend.proxy;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.Proxy;
|
||||
import java.net.ProxySelector;
|
||||
import java.net.SocketAddress;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
|
||||
public class TalendProxySelector extends ProxySelector {
|
||||
private static TalendProxySelector instance;
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(TalendProxySelector.class);
|
||||
private static final Logger log = Logger.getLogger(TalendProxySelector.class);
|
||||
|
||||
private ThreadLocal<ProxyHolder> threadLocalProxyHolder;
|
||||
private ProxyHolder globalProxyHolder;
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend-webservice-input</artifactId>
|
||||
<version>1.0.2</version>
|
||||
<version>1.0.0-20170301</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend-webservice-input</name>
|
||||
@@ -51,12 +51,6 @@
|
||||
<groupId>commons-discovery</groupId>
|
||||
<artifactId>commons-discovery</artifactId>
|
||||
<version>0.2</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/wsdl4j/wsdl4j -->
|
||||
<dependency>
|
||||
@@ -70,6 +64,12 @@
|
||||
<groupId>javax.xml</groupId>
|
||||
<artifactId>jaxrpc-api</artifactId>
|
||||
<version>1.1</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/commons-logging/commons-logging -->
|
||||
<dependency>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
<version>1.1</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -95,7 +95,6 @@
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<version>2.6</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>default-jar</id>
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
<pathelement location="${component.plugin.home}/tMicrosoftCrmInput/commons-logging-1.1.1.jar" />
|
||||
<pathelement location="${component.plugin.home}/tWriteJSONFieldIn/commons-lang-2.6.jar" />
|
||||
<pathelement location="${component.plugin.home}/tWebService/commons-collections-3.2.1.jar" />
|
||||
<pathelement location="${component.plugin.home}/tWebService/commons-beanutils-1.9.4.jar" />
|
||||
<pathelement location="${component.plugin.home}/tWebService/commons-beanutils-1.7.0.jar" />
|
||||
<pathelement location="${component.plugin.home}/tWebService/cxf-2.6.0-modified.jar" />
|
||||
<pathelement location="${component.plugin.home}/tWebService/jaxb-xjc-2.2.5-modified.jar" />
|
||||
<pathelement location="${component.plugin.home}/tWebService/jaxb-impl-2.2.5-modified.jar" />
|
||||
|
||||
@@ -85,7 +85,7 @@
|
||||
<dependency>
|
||||
<groupId>commons-beanutils</groupId>
|
||||
<artifactId>commons-beanutils</artifactId>
|
||||
<version>1.9.4</version>
|
||||
<version>1.8.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.cxf</groupId>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>talendsap</artifactId>
|
||||
<name>talendsap</name>
|
||||
<version>1.0.3</version>
|
||||
<version>1.0.2</version>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
@@ -35,15 +35,9 @@
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.dom4j</groupId>
|
||||
<groupId>dom4j</groupId>
|
||||
<artifactId>dom4j</artifactId>
|
||||
<version>2.1.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>jaxen</groupId>
|
||||
<artifactId>jaxen</artifactId>
|
||||
<version>1.1.6</version>
|
||||
<version>1.6.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
@@ -16,11 +16,21 @@ public class DocumentExtractor {
|
||||
|
||||
public DocumentExtractor(Document doc,String function) {
|
||||
this.doc = doc;
|
||||
this.finalFunction = SAPXMLCoder.encode(function);
|
||||
this.finalFunction = replaceNamespace(function);
|
||||
}
|
||||
|
||||
/**
|
||||
* the xml document which come from the cimt sap api will change the element name which contains "/" and convert "/" to "_-",
|
||||
* we need to do the same thing for finding the element by the xpath
|
||||
* @param name
|
||||
* @return
|
||||
*/
|
||||
private String replaceNamespace(String name) {
|
||||
return name.replace("/", "_-");
|
||||
}
|
||||
|
||||
public String getSingleResult(String name) {
|
||||
final String finalName = SAPXMLCoder.encode(name);
|
||||
final String finalName = replaceNamespace(name);
|
||||
|
||||
XPath xpath = org.dom4j.DocumentHelper.createXPath(
|
||||
sb.append("/").append(finalFunction).append("/OUTPUT/").append(finalName)
|
||||
@@ -37,7 +47,7 @@ public class DocumentExtractor {
|
||||
}
|
||||
|
||||
public List<String> getStructureResult(String structureName,List<String> names) {
|
||||
final String finalStructureName = SAPXMLCoder.encode(structureName);
|
||||
final String finalStructureName = replaceNamespace(structureName);
|
||||
|
||||
XPath xpath = org.dom4j.DocumentHelper.createXPath(
|
||||
sb.append("/").append(finalFunction).append("/OUTPUT/").append(finalStructureName)
|
||||
@@ -55,7 +65,7 @@ public class DocumentExtractor {
|
||||
List<String> result = new ArrayList<String>();
|
||||
|
||||
for(String name : names) {
|
||||
final String finalName = SAPXMLCoder.encode(name);
|
||||
final String finalName = replaceNamespace(name);
|
||||
Node subNode = node.selectSingleNode(finalName);
|
||||
if(subNode == null) {
|
||||
result.add(null);
|
||||
@@ -68,7 +78,7 @@ public class DocumentExtractor {
|
||||
}
|
||||
|
||||
public List<List<String>> getTableResult(String tableName,List<String> names) {
|
||||
final String finalTableName = SAPXMLCoder.encode(tableName);
|
||||
final String finalTableName = replaceNamespace(tableName);
|
||||
|
||||
List<List<String>> result = new ArrayList<List<String>>();
|
||||
|
||||
@@ -102,7 +112,7 @@ public class DocumentExtractor {
|
||||
for(Element element : elements) {
|
||||
List<String> row = new ArrayList<String>();
|
||||
for(String name : names) {
|
||||
final String finalName = SAPXMLCoder.encode(name);
|
||||
final String finalName = replaceNamespace(name);
|
||||
Element subElement = element.element(finalName);
|
||||
if(subElement == null) {
|
||||
row.add(null);
|
||||
|
||||
@@ -28,7 +28,7 @@ public class DocumentHelper {
|
||||
}
|
||||
|
||||
public void setFunctionName(String name) {
|
||||
root = doc.addElement(SAPXMLCoder.encode(name));
|
||||
root = doc.addElement(name);
|
||||
}
|
||||
|
||||
private void correctInput() {
|
||||
@@ -55,20 +55,20 @@ public class DocumentHelper {
|
||||
}
|
||||
if (parameter_type == SAPParameterType.CHANGING) {
|
||||
correctChanging();
|
||||
changing.addElement(SAPXMLCoder.encode(name)).setText(value);
|
||||
changing.addElement(name).setText(value);
|
||||
} else {
|
||||
correctInput();
|
||||
input.addElement(SAPXMLCoder.encode(name)).setText(value);
|
||||
input.addElement(name).setText(value);
|
||||
}
|
||||
}
|
||||
|
||||
public void addStructParameter(String name, SAPParameterType parameter_type) {
|
||||
if (parameter_type == SAPParameterType.CHANGING) {
|
||||
correctChanging();
|
||||
currentStruct = changing.addElement(SAPXMLCoder.encode(name));
|
||||
currentStruct = changing.addElement(name);
|
||||
} else {
|
||||
correctInput();
|
||||
currentStruct = input.addElement(SAPXMLCoder.encode(name));
|
||||
currentStruct = input.addElement(name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,19 +76,19 @@ public class DocumentHelper {
|
||||
if(value == null) {
|
||||
value = "";
|
||||
}
|
||||
currentStruct.addElement(SAPXMLCoder.encode(name)).setText(value);
|
||||
currentStruct.addElement(name).setText(value);
|
||||
}
|
||||
|
||||
public void addTableParameter(String name, SAPParameterType parameter_type) {
|
||||
if (parameter_type == SAPParameterType.CHANGING) {
|
||||
correctChanging();
|
||||
currentTable = changing.addElement(SAPXMLCoder.encode(name));
|
||||
currentTable = changing.addElement(name);
|
||||
} else if(parameter_type == SAPParameterType.TABLES) {
|
||||
correctTables();
|
||||
currentTable = tables.addElement(SAPXMLCoder.encode(name));
|
||||
currentTable = tables.addElement(name);
|
||||
} else {
|
||||
correctInput();
|
||||
currentTable = input.addElement(SAPXMLCoder.encode(name));
|
||||
currentTable = input.addElement(name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,11 +100,66 @@ public class DocumentHelper {
|
||||
if(value == null) {
|
||||
value = "";
|
||||
}
|
||||
currentRow.addElement(SAPXMLCoder.encode(name)).setText(value);
|
||||
currentRow.addElement(name).setText(value);
|
||||
}
|
||||
|
||||
public Document getDocument() {
|
||||
return doc;
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
DocumentHelper helper = new DocumentHelper();
|
||||
helper.setFunctionName("READ_TABLE_FUNCTION");
|
||||
|
||||
helper.addSingleParameter("ID", "1", SAPParameterType.CHANGING);
|
||||
helper.addSingleParameter("NAME", "gaoyan", SAPParameterType.IMPORT);
|
||||
|
||||
helper.addStructParameter("INFO", SAPParameterType.CHANGING);
|
||||
helper.addStructChildParameter("ID", "2");
|
||||
helper.addStructChildParameter("NAME", "wangwei");
|
||||
|
||||
helper.addStructParameter("INFO1", SAPParameterType.IMPORT);
|
||||
helper.addStructChildParameter("ID1", "4");
|
||||
helper.addStructChildParameter("NAME1", "momo");
|
||||
|
||||
helper.addTableParameter("TABLE1", SAPParameterType.TABLES);
|
||||
for (int i = 0; i < 200000; i++) {
|
||||
helper.addTableRow();
|
||||
helper.addTableRowChildParameter("c1", i + "");
|
||||
helper.addTableRowChildParameter("c2", "wangwei" + i);
|
||||
helper.addTableRowChildParameter("c3", "wangwei" + i);
|
||||
helper.addTableRowChildParameter("c4", "wangwei" + i);
|
||||
helper.addTableRowChildParameter("c5", "wangwei" + i);
|
||||
helper.addTableRowChildParameter("c6", "wangwei" + i);
|
||||
helper.addTableRowChildParameter("c7", "wangwei" + i);
|
||||
helper.addTableRowChildParameter("c8", "wangwei" + i);
|
||||
}
|
||||
|
||||
helper.addTableParameter("TABLE2", SAPParameterType.TABLES);
|
||||
for (int i = 0; i < 2; i++) {
|
||||
helper.addTableRow();
|
||||
helper.addTableRowChildParameter("ID4", i + "");
|
||||
helper.addTableRowChildParameter("NAME4", "gaoyan" + i);
|
||||
}
|
||||
|
||||
// StringWriter sw = new StringWriter();
|
||||
// OutputFormat format = OutputFormat.createPrettyPrint();
|
||||
// XMLWriter writer = new XMLWriter(sw, format);
|
||||
Document doc = helper.getDocument();
|
||||
// writer.write(doc);
|
||||
// System.out.println(sw.toString());
|
||||
|
||||
DocumentExtractor ext = new DocumentExtractor(doc,"READ_TABLE_FUNCTION");
|
||||
String single = ext.getSingleResult("ID");
|
||||
System.out.println(single);
|
||||
|
||||
List<String> struct = ext.getStructureResult("INFO", Arrays.asList("ID","NAME"));
|
||||
System.out.println(struct);
|
||||
|
||||
List<List<String>> table = ext.getTableResult("TABLE1", Arrays.asList("c1","c2","c3","c4","c5","c6","c7","c8"));
|
||||
for(List<String> row : table) {
|
||||
System.out.println(row);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
package org.talend.sap;
|
||||
|
||||
/**
|
||||
* sap <-> valid xml element name by coder
|
||||
*
|
||||
* first char :
|
||||
* 0 <-> _0
|
||||
* 1 <-> _1
|
||||
* 2 <-> _2
|
||||
* ...
|
||||
* 9 <-> _9
|
||||
*
|
||||
* any location char
|
||||
* / <-> _-
|
||||
*/
|
||||
public class SAPXMLCoder {
|
||||
|
||||
/**
|
||||
* convert sapname to xmlname
|
||||
* @param sapname
|
||||
* @return
|
||||
*/
|
||||
public static String encode(String sapname) {
|
||||
if(sapname == null) {
|
||||
return sapname;
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for(int i=0;i<sapname.length();i++) {
|
||||
char c = sapname.charAt(i);
|
||||
|
||||
if(i==0) {
|
||||
if(isDigit(c)) {
|
||||
sb.append('_').append(c);
|
||||
} else if(c == '/'){
|
||||
sb.append("_-");
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
} else {
|
||||
if(c == '/'){
|
||||
sb.append("_-");
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* convert xmlname to sapname
|
||||
* @param xmlname
|
||||
* @return
|
||||
*/
|
||||
static String decode(String xmlname) {
|
||||
if(xmlname == null) {
|
||||
return xmlname;
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for(int i=0;i<xmlname.length();i++) {
|
||||
char c = xmlname.charAt(i);
|
||||
|
||||
if(i==0) {
|
||||
if(c == '_') {
|
||||
int next_index = i + 1;
|
||||
if(next_index < xmlname.length()) {
|
||||
char next = xmlname.charAt(next_index);
|
||||
if(isDigit(next)) {
|
||||
sb.append(next);
|
||||
i++;
|
||||
} else if(next == '-') {
|
||||
sb.append('/');
|
||||
i++;
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
} else {
|
||||
if(c == '_') {
|
||||
int next_index = i + 1;
|
||||
if(next_index < xmlname.length()) {
|
||||
char next = xmlname.charAt(next_index);
|
||||
if(next == '-') {
|
||||
sb.append('/');
|
||||
i++;
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static boolean isDigit(char c) {
|
||||
return c>=48 && c<=57;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
//ecnode
|
||||
System.out.println(encode("/CMT/TLND_TABLE"));
|
||||
System.out.println(encode("0CMT0TLND_TABLE"));
|
||||
System.out.println(encode("1CMT1TLND_TABLE"));
|
||||
|
||||
//decode
|
||||
System.out.println(decode(encode("/CMT/TLND_TABLE")));
|
||||
System.out.println(decode(encode("0CMT0TLND_TABLE")));
|
||||
System.out.println(decode(encode("1CMT1TLND_TABLE")));
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package org.talend.sap;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.dom4j.Document;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
public class BasicTest {
|
||||
|
||||
@Test
|
||||
public void testSAPXMLCoder() {
|
||||
assertEquals("_-CMT_-TLND_TABLE", SAPXMLCoder.encode("/CMT/TLND_TABLE"));
|
||||
assertEquals("_0CMT0TLND_TABLE", SAPXMLCoder.encode("0CMT0TLND_TABLE"));
|
||||
|
||||
assertEquals("/CMT/TLND_TABLE", SAPXMLCoder.decode(SAPXMLCoder.encode("/CMT/TLND_TABLE")));
|
||||
assertEquals("0CMT0TLND_TABLE", SAPXMLCoder.decode(SAPXMLCoder.encode("0CMT0TLND_TABLE")));
|
||||
|
||||
assertEquals("BAPI_MATERIAL_GET_ALL", SAPXMLCoder.decode(SAPXMLCoder.encode("BAPI_MATERIAL_GET_ALL")));
|
||||
assertEquals("2STEP_PICK", SAPXMLCoder.decode(SAPXMLCoder.encode("2STEP_PICK")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDocumentHelperAndExtractor() {
|
||||
DocumentHelper helper = new DocumentHelper();
|
||||
String function = "/CMT/TLND_TABLE";
|
||||
helper.setFunctionName(function);
|
||||
|
||||
helper.addSingleParameter("ID", "1", SAPParameterType.CHANGING);
|
||||
helper.addSingleParameter("NAME", "gaoyan", SAPParameterType.IMPORT);
|
||||
|
||||
helper.addStructParameter("INFO", SAPParameterType.CHANGING);
|
||||
helper.addStructChildParameter("ID", "2");
|
||||
helper.addStructChildParameter("NAME", "wangwei");
|
||||
|
||||
helper.addStructParameter("INFO1", SAPParameterType.IMPORT);
|
||||
helper.addStructChildParameter("ID1", "4");
|
||||
helper.addStructChildParameter("NAME1", "momo");
|
||||
|
||||
helper.addTableParameter("TABLE1", SAPParameterType.TABLES);
|
||||
for (int i = 0; i < 10; i++) {
|
||||
helper.addTableRow();
|
||||
helper.addTableRowChildParameter("c1", i + "");
|
||||
helper.addTableRowChildParameter("c2", "wangwei" + i);
|
||||
helper.addTableRowChildParameter("c3", "wangwei" + i);
|
||||
helper.addTableRowChildParameter("c4", "wangwei" + i);
|
||||
helper.addTableRowChildParameter("c5", "wangwei" + i);
|
||||
helper.addTableRowChildParameter("c6", "wangwei" + i);
|
||||
helper.addTableRowChildParameter("c7", "wangwei" + i);
|
||||
helper.addTableRowChildParameter("c8", "wangwei" + i);
|
||||
}
|
||||
|
||||
helper.addTableParameter("TABLE2", SAPParameterType.TABLES);
|
||||
for (int i = 0; i < 2; i++) {
|
||||
helper.addTableRow();
|
||||
helper.addTableRowChildParameter("ID4", i + "");
|
||||
helper.addTableRowChildParameter("NAME4", "gaoyan" + i);
|
||||
}
|
||||
|
||||
Document doc = helper.getDocument();
|
||||
|
||||
DocumentExtractor ext = new DocumentExtractor(doc,function);
|
||||
String single = ext.getSingleResult("ID");
|
||||
assertEquals("1", single);
|
||||
|
||||
List<String> struct = ext.getStructureResult("INFO", Arrays.asList("ID","NAME"));
|
||||
assertEquals(java.util.Arrays.asList("2", "wangwei"), struct);
|
||||
|
||||
List<List<String>> table = ext.getTableResult("TABLE1", Arrays.asList("c1","c2","c3","c4","c5","c6","c7","c8"));
|
||||
assertEquals(10, table.size());
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>org.talend.studio</groupId>
|
||||
<artifactId>tdi-studio-se</artifactId>
|
||||
<version>7.3.1-PATCH</version>
|
||||
<version>7.3.1</version>
|
||||
<relativePath>../../../</relativePath>
|
||||
</parent>
|
||||
<artifactId>org.talend.designer.components.libs</artifactId>
|
||||
|
||||
@@ -132,7 +132,7 @@ int tmp_batchUpdateCount_<%=cid%> = 0;
|
||||
int deletedCount_<%=cid%>=0;
|
||||
int updatedCount_<%=cid%>=0;
|
||||
int insertedCount_<%=cid%>=0;
|
||||
int rowsToCommitCount_<%=cid%>=0;
|
||||
|
||||
String tableName_<%=cid%> = <%=table%>;
|
||||
boolean whetherReject_<%=cid%> = false;
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ imports="
|
||||
org.talend.core.model.process.INode
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
org.talend.core.model.process.IConnection
|
||||
org.talend.core.model.process.IProcess
|
||||
java.util.List
|
||||
"
|
||||
%>
|
||||
@@ -15,15 +14,11 @@ imports="
|
||||
|
||||
String cid = node.getUniqueName();
|
||||
dbLog = new DBLogUtil(node);
|
||||
IProcess process = node.getProcess();
|
||||
|
||||
String dataAction = ElementParameterParser.getValue(node,"__DATA_ACTION__");
|
||||
|
||||
String commitEvery = ElementParameterParser.getValue(node, "__COMMIT_EVERY__");
|
||||
|
||||
boolean setAutoCommit = false;
|
||||
|
||||
boolean useExistingConnection = "true".equals(ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__"));
|
||||
String useExistingConn = ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__");
|
||||
|
||||
String useCommitControl = ElementParameterParser.getValue(node,"__USE_COMMIT_CONTROL__");
|
||||
|
||||
@@ -35,17 +30,6 @@ imports="
|
||||
IConnection rejectConn = rejectConns.get(0);
|
||||
rejectConnName = rejectConn.getName();
|
||||
}
|
||||
|
||||
String connection = ElementParameterParser.getValue(node,"__CONNECTION__");
|
||||
if(useExistingConnection) {
|
||||
List<? extends INode> nodes = process.getNodesOfType("tAS400Connection");
|
||||
for (INode ne:nodes) {
|
||||
if (connection.equals(ne.getUniqueName())) {
|
||||
setAutoCommit = "true".equals(ElementParameterParser.getValue(ne, "__AUTO_COMMIT__"));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
boolean useBatch = (rejectConnName == null)
|
||||
&& (
|
||||
@@ -98,16 +82,14 @@ imports="
|
||||
%>
|
||||
resourceMap.put("statementClosed_<%=cid%>", true);
|
||||
<%
|
||||
if(!useExistingConnection) {
|
||||
if(!("true").equals(useExistingConn)) {
|
||||
if(("true").equals(useCommitControl)) {
|
||||
if(!("").equals(commitEvery) && !("0").equals(commitEvery)) {
|
||||
%>
|
||||
if(commitCounter_<%=cid%> > 0 && rowsToCommitCount_<%=cid%> != 0) {
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
if(commitCounter_<%=cid%> > 0) {
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
commitCounter_<%=cid%> = 0;
|
||||
}
|
||||
<%
|
||||
}
|
||||
|
||||
@@ -310,9 +310,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.sql.SQLException e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
@@ -444,9 +442,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.sql.SQLException e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
@@ -593,9 +589,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.sql.SQLException e) {
|
||||
@@ -671,9 +665,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmtInsert_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.sql.SQLException e) {
|
||||
@@ -803,7 +795,6 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
updateFlag_<%=cid%>=pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%>+updateFlag_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += updateFlag_<%=cid%>;
|
||||
|
||||
if(updateFlag_<%=cid%> == 0) {
|
||||
|
||||
@@ -849,9 +840,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmtInsert_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
}else{
|
||||
@@ -941,9 +930,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
deletedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
deletedCount_<%=cid%> = deletedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().deleting(dbLog.var("nb_line"));%>
|
||||
} catch(java.sql.SQLException e) {
|
||||
@@ -1027,13 +1014,10 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
<%
|
||||
}%>
|
||||
if(rowsToCommitCount_<%=cid%> != 0){
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
}
|
||||
commitCounter_<%=cid%>=0;
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
commitCounter_<%=cid%>=0;
|
||||
}
|
||||
<%
|
||||
}
|
||||
|
||||
@@ -100,7 +100,7 @@ int nb_line_rejected_<%=cid%> = 0;
|
||||
int deletedCount_<%=cid%>=0;
|
||||
int updatedCount_<%=cid%>=0;
|
||||
int insertedCount_<%=cid%>=0;
|
||||
int rowsToCommitCount_<%=cid%>=0;
|
||||
|
||||
String tableName_<%=cid%> = <%=tableName%>;
|
||||
boolean whetherReject_<%=cid%> = false;
|
||||
java.sql.Connection conn_<%=cid%> = null;
|
||||
|
||||
@@ -17,7 +17,6 @@ imports="
|
||||
String dataAction = ElementParameterParser.getValue(node,"__DATA_ACTION__");
|
||||
|
||||
String commitEvery = ElementParameterParser.getValue(node, "__COMMIT_EVERY__");
|
||||
boolean setAutoCommit = false;
|
||||
boolean useExistingConnection = "true".equals(ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__"));
|
||||
|
||||
if(("INSERT_OR_UPDATE").equals(dataAction)) {
|
||||
@@ -61,13 +60,9 @@ imports="
|
||||
{
|
||||
if(!("").equals(commitEvery)&&!("0").equals(commitEvery)){
|
||||
%>
|
||||
if(rowsToCommitCount_<%=cid%> != 0){
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
}
|
||||
commitCounter_<%=cid%> = 0;
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -140,9 +140,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -232,9 +230,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -352,9 +348,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -416,9 +410,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try{
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmtInsert_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> = processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -519,8 +511,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
}
|
||||
updatedCount_<%=cid%> += updateFlag_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += updateFlag_<%=cid%>;
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%>+updateFlag_<%=cid%>;
|
||||
} catch(java.lang.Exception e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
<%
|
||||
@@ -582,9 +573,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmtInsert_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -596,7 +585,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
nb_line_<%=cid%>++;
|
||||
nb_line_<%=cid%>++;
|
||||
<%
|
||||
if(rejectConnName != null && rejectColumnList != null && rejectColumnList.size() > 0) {
|
||||
%>
|
||||
@@ -663,8 +652,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
deletedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
deletedCount_<%=cid%> = deletedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().deleting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -709,7 +697,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
<%
|
||||
for(IConnection outgoingConn : outgoingConns) {
|
||||
if(rejectConnName == null || (rejectConnName != null && !outgoingConn.getName().equals(rejectConnName))) {
|
||||
if(outgoingConn.getLineStyle().hasConnectionCategory(IConnectionCategory.DATA)){
|
||||
if(outgoingConn.getLineStyle().hasConnectionCategory(IConnectionCategory.DATA)) {
|
||||
%>
|
||||
<%=outgoingConn.getName()%> = new <%=outgoingConn.getName()%>Struct();
|
||||
<%
|
||||
@@ -732,15 +720,12 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
commitCounter_<%=cid%>++;
|
||||
|
||||
if(commitEvery_<%=cid%> <= commitCounter_<%=cid%>){
|
||||
if(rowsToCommitCount_<%=cid%> != 0){
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
}
|
||||
commitCounter_<%=cid%>=0;
|
||||
}
|
||||
if(commitEvery_<%=cid%><=commitCounter_<%=cid%>) {
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
commitCounter_<%=cid%>=0;
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
@@ -465,7 +465,7 @@ class GenerateToolByDom4j{
|
||||
}
|
||||
%>
|
||||
}
|
||||
<%touchXMLNode.getXMLElement(parentName);%>.elements().add(orders_<%=cid %>[<%=currPos %>],<%touchXMLNode.getXMLElement(currEleName);%>);
|
||||
<%touchXMLNode.getXMLElement(parentName);%>.elements().add(orders_<%=cid %>[<%=currPos %>],<%touchXMLNode.getXMLNode(currEleName);%>);
|
||||
<%
|
||||
}else{
|
||||
%>
|
||||
@@ -475,7 +475,7 @@ class GenerateToolByDom4j{
|
||||
if(<%=currPos +1 %> < orders_<%=cid %>.length){
|
||||
orders_<%=cid %>[<%=currPos +1 %>] = 0;
|
||||
}
|
||||
<%touchXMLNode.getXMLElement(parentName);%>.elements().add(orders_<%=cid %>[<%=currPos %>]++,<%touchXMLNode.getXMLElement(currEleName);%>);
|
||||
<%touchXMLNode.getXMLElement(parentName);%>.elements().add(orders_<%=cid %>[<%=currPos %>]++,<%touchXMLNode.getXMLNode(currEleName);%>);
|
||||
<%
|
||||
}
|
||||
}else{
|
||||
|
||||
@@ -5,7 +5,7 @@ if(!use_customize_version){
|
||||
%>String[] applications_<%=cid%> = <%
|
||||
String application = ElementParameterParser.getValue(node, "__APPLICATION__");
|
||||
if("ALL_APPLICATIONS".equals(application)) {
|
||||
if("\"emr-4.6.0\",\"emr-4.9.2\",\"emr-5.11.0\",\"emr-5.15.0\", \"emr-5.29.0\"".contains(cluster_version)) {
|
||||
if("\"emr-4.6.0\",\"emr-4.9.2\",\"emr-5.11.0\",\"emr-5.15.0\"".contains(cluster_version)) {
|
||||
%>
|
||||
{}
|
||||
<%
|
||||
@@ -15,7 +15,7 @@ if(!use_customize_version){
|
||||
<%
|
||||
}
|
||||
} else if("CORE_HADOOP".equals(application)) {
|
||||
if("\"emr-5.11.0\"".equals(cluster_version) || "\"emr-5.15.0\"".equals(cluster_version) || "\"emr-5.29.0\"".equals(cluster_version)){%>
|
||||
if("\"emr-5.11.0\"".equals(cluster_version) || "\"emr-5.15.0\"".equals(cluster_version)){%>
|
||||
{"Ganglia","Hadoop","Hive","Hue","Mahout","Pig","Tez"}
|
||||
<%
|
||||
}else if("\"emr-4.6.0\"".equals(cluster_version) || "\"emr-4.9.2\"".equals(cluster_version)) {
|
||||
@@ -38,7 +38,7 @@ if(!use_customize_version){
|
||||
<%
|
||||
}
|
||||
} else if("PRESTO".equals(application)) {
|
||||
if("\"emr-5.11.0\"".equals(cluster_version) || "\"emr-5.15.0\"".equals(cluster_version) || "\"emr-5.29.0\"".equals(cluster_version)){%>
|
||||
if("\"emr-5.11.0\"".equals(cluster_version) || "\"emr-5.15.0\"".equals(cluster_version)){%>
|
||||
{"Hadoop","Hive","Presto"}
|
||||
<% } else { %>
|
||||
if(true) {
|
||||
@@ -47,7 +47,7 @@ if(!use_customize_version){
|
||||
<%
|
||||
}
|
||||
} else if("SPARK".equals(application)) {
|
||||
if("\"emr-5.11.0\"".equals(cluster_version) || "\"emr-5.15.0\"".equals(cluster_version) || "\"emr-5.29.0\"".equals(cluster_version)){%>
|
||||
if("\"emr-5.11.0\"".equals(cluster_version) || "\"emr-5.15.0\"".equals(cluster_version)){%>
|
||||
{"Ganglia","Hadoop","Spark","Zeppelin"}
|
||||
<%
|
||||
} else if("\"emr-4.6.0\"".equals(cluster_version) || "\"emr-4.9.2\"".equals(cluster_version)) {
|
||||
@@ -60,7 +60,7 @@ if(!use_customize_version){
|
||||
<%
|
||||
}
|
||||
} else if("HBASE".equals(application)) {
|
||||
if("\"emr-5.11.0\"".equals(cluster_version) || "\"emr-4.9.2\"".equals(cluster_version) || "\"emr-5.15.0\"".equals(cluster_version) || "\"emr-5.29.0\"".equals(cluster_version)){%>
|
||||
if("\"emr-5.11.0\"".equals(cluster_version) || "\"emr-4.9.2\"".equals(cluster_version) || "\"emr-5.15.0\"".equals(cluster_version)){%>
|
||||
{"HBase","Ganglia","Hadoop","Hive","Hue","Phoenix","ZooKeeper"}
|
||||
<%
|
||||
} else if("\"emr-4.6.0\"".equals(cluster_version)) {
|
||||
|
||||
@@ -403,7 +403,7 @@
|
||||
if(wait_for_cluster_ready) {
|
||||
%>
|
||||
boolean clusterIsAlive_<%=cid%> = false;
|
||||
System.out.println("Waiting for cluster to become available.");
|
||||
System.out.println("Wating for cluster to become available.");
|
||||
while (true) {
|
||||
com.amazonaws.services.elasticmapreduce.model.DescribeClusterResult result2_<%=cid%> = emr_<%=cid%>.describeCluster(
|
||||
new com.amazonaws.services.elasticmapreduce.model.DescribeClusterRequest()
|
||||
@@ -433,7 +433,7 @@
|
||||
boolean waitForStep_<%=cid%> = clusterIsAlive_<%=cid%> && (steps_<%=cid %>.size() > 1);
|
||||
|
||||
if(waitForStep_<%=cid%>) {
|
||||
System.out.println("Waiting for steps to complete.");
|
||||
System.out.println("Wating for steps to complete.");
|
||||
}
|
||||
|
||||
while (waitForStep_<%=cid%>) {
|
||||
|
||||
@@ -97,8 +97,7 @@
|
||||
|
||||
<PARAMETER NAME="CLUSTER_VERSION" FIELD="OPENED_LIST"
|
||||
NUM_ROW="32" SHOW_IF="(ACTION=='START') AND (USE_CUSTOMIZE_VERSION == 'false')" GROUP="CONFIGURATION">
|
||||
<ITEMS DEFAULT="EMR5_29_0">
|
||||
<ITEM NAME="EMR5_29_0" VALUE=""emr-5.29.0"" />
|
||||
<ITEMS DEFAULT="EMR5_15_0">
|
||||
<ITEM NAME="EMR5_15_0" VALUE=""emr-5.15.0"" />
|
||||
<ITEM NAME="EMR5_11_0" VALUE=""emr-5.11.0"" />
|
||||
<ITEM NAME="EMR4_9_2" VALUE=""emr-4.9.2"" />
|
||||
@@ -113,14 +112,14 @@
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="APPLICATION" FIELD="CLOSED_LIST" NUM_ROW="32"
|
||||
SHOW_IF="(ACTION=='START') AND (USE_CUSTOMIZE_VERSION == 'false') AND ((CLUSTER_VERSION=='EMR5_29_0') OR (CLUSTER_VERSION=='EMR5_15_0') OR (CLUSTER_VERSION=='EMR5_11_0') OR (CLUSTER_VERSION=='EMR4_9_2') OR (CLUSTER_VERSION=='EMR4_6_0') OR (CLUSTER_VERSION=='EMR4_5_0'))" GROUP="CONFIGURATION">
|
||||
SHOW_IF="(ACTION=='START') AND (USE_CUSTOMIZE_VERSION == 'false') AND ((CLUSTER_VERSION=='EMR5_15_0') OR (CLUSTER_VERSION=='EMR5_11_0') OR (CLUSTER_VERSION=='EMR4_9_2') OR (CLUSTER_VERSION=='EMR4_6_0') OR (CLUSTER_VERSION=='EMR4_5_0'))" GROUP="CONFIGURATION">
|
||||
|
||||
<ITEMS DEFAULT="CORE_HADOOP">
|
||||
<ITEM NAME="ALL_APPLICATIONS" VALUE="ALL_APPLICATIONS" NOT_SHOW_IF="(CLUSTER_VERSION=='EMR4_6_0') OR (CLUSTER_VERSION=='EMR4_9_2') OR (CLUSTER_VERSION=='EMR5_11_0') OR (CLUSTER_VERSION=='EMR5_15_0') OR (CLUSTER_VERSION=='EMR5_29_0')"/>
|
||||
<ITEM NAME="ALL_APPLICATIONS" VALUE="ALL_APPLICATIONS" NOT_SHOW_IF="(CLUSTER_VERSION=='EMR4_6_0') OR (CLUSTER_VERSION=='EMR4_9_2') OR (CLUSTER_VERSION=='EMR5_11_0') OR (CLUSTER_VERSION=='EMR5_15_0')"/>
|
||||
<ITEM NAME="CORE_HADOOP" VALUE="CORE_HADOOP" />
|
||||
<ITEM NAME="HBASE" VALUE="HBASE" SHOW_IF="(CLUSTER_VERSION=='EMR5_29_0') OR (CLUSTER_VERSION=='EMR5_11_0') OR (CLUSTER_VERSION=='EMR4_9_2') OR (CLUSTER_VERSION=='EMR4_6_0') OR (CLUSTER_VERSION=='EMR5_15_0')"/>
|
||||
<ITEM NAME="PRESTO_SANDBOX" VALUE="PRESTO_SANDBOX" NOT_SHOW_IF="(CLUSTER_VERSION=='EMR5_11_0') OR (CLUSTER_VERSION=='EMR5_15_0') OR (CLUSTER_VERSION=='EMR5_29_0')"/>
|
||||
<ITEM NAME="PRESTO" VALUE="PRESTO" SHOW_IF="(CLUSTER_VERSION=='EMR5_11_0') OR (CLUSTER_VERSION=='EMR5_15_0') OR (CLUSTER_VERSION=='EMR5_29_0')"/>
|
||||
<ITEM NAME="HBASE" VALUE="HBASE" SHOW_IF="(CLUSTER_VERSION=='EMR5_11_0') OR (CLUSTER_VERSION=='EMR4_9_2') OR (CLUSTER_VERSION=='EMR4_6_0') OR (CLUSTER_VERSION=='EMR5_15_0')"/>
|
||||
<ITEM NAME="PRESTO_SANDBOX" VALUE="PRESTO_SANDBOX" NOT_SHOW_IF="(CLUSTER_VERSION=='EMR5_11_0') OR (CLUSTER_VERSION=='EMR5_15_0')"/>
|
||||
<ITEM NAME="PRESTO" VALUE="PRESTO" SHOW_IF="(CLUSTER_VERSION=='EMR5_11_0') OR (CLUSTER_VERSION=='EMR5_15_0')"/>
|
||||
<ITEM NAME="SPARK" VALUE="SPARK" />
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
@@ -407,7 +406,7 @@
|
||||
|
||||
<PARAMETER NAME="CUSTOMIZE_VERSION" FIELD="TEXT"
|
||||
NUM_ROW="35" SHOW_IF="(ACTION=='START') AND (USE_CUSTOMIZE_VERSION =='true')" GROUP="CUSTOMIZE_APPLICATION_GROUP">
|
||||
<DEFAULT>"emr-5.29.0"</DEFAULT>
|
||||
<DEFAULT>"emr-5.15.0"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="CUSTOMIZE_APPLICATION_TABLE" FIELD="TABLE" NUM_ROW="40" NB_LINES="5" GROUP="CUSTOMIZE_APPLICATION_GROUP"
|
||||
|
||||
@@ -29,7 +29,6 @@ REGION.ITEM.SA_EAST_1=South America (Sao Paulo)
|
||||
CLUSTER_NAME.NAME=Cluster name
|
||||
|
||||
CLUSTER_VERSION.NAME=Cluster version
|
||||
CLUSTER_VERSION.ITEM.EMR5_29_0=EMR 5.29.0
|
||||
CLUSTER_VERSION.ITEM.EMR5_15_0=EMR 5.15.0
|
||||
CLUSTER_VERSION.ITEM.EMR5_11_0=EMR 5.11.0
|
||||
CLUSTER_VERSION.ITEM.EMR4_9_2=EMR 4.9.2
|
||||
|
||||
@@ -163,7 +163,7 @@ int nb_line_rejected_<%=cid%> = 0;
|
||||
int deletedCount_<%=cid%>=0;
|
||||
int updatedCount_<%=cid%>=0;
|
||||
int insertedCount_<%=cid%>=0;
|
||||
int rowsToCommitCount_<%=cid%>=0;
|
||||
|
||||
int rejectedCount_<%=cid%>=0;
|
||||
|
||||
String tableName_<%=cid%> = <%=table%>;
|
||||
|
||||
@@ -38,9 +38,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
|
||||
String commitEvery = ElementParameterParser.getValue(node, "__COMMIT_EVERY__");
|
||||
|
||||
boolean setAutoCommit = false;
|
||||
|
||||
boolean useExistingConnection = "true".equals(ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__"));
|
||||
String useExistingConn = ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__");
|
||||
|
||||
boolean extendedInsert = false;
|
||||
|
||||
@@ -52,13 +50,11 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
|
||||
String dbVersion = "" ;
|
||||
String connection = ElementParameterParser.getValue(node,"__CONNECTION__");
|
||||
if(useExistingConnection) {
|
||||
if(("true").equals(useExistingConn)) {
|
||||
List<? extends INode> nodes = process.getNodesOfType("tAmazonMysqlConnection");
|
||||
for (INode ne:nodes) {
|
||||
if (connection.equals(ne.getUniqueName())) {
|
||||
dbVersion = ElementParameterParser.getValue(ne, "__DB_VERSION__");
|
||||
setAutoCommit = "true".equals(ElementParameterParser.getValue(ne, "__AUTO_COMMIT__"));
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -121,7 +117,6 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
for(int countEach_<%=cid%>: pstmt_<%=cid %>.executeBatch()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : 1);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%dbLog.batch().executeDone(dbLog.str(dataAction));%>
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
@@ -142,7 +137,6 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
for(int countEach_<%=cid%>: e.getUpdateCounts()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : countEach_<%=cid%>);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
insertedCount_<%=cid%> += countSum_<%=cid%>;
|
||||
@@ -198,7 +192,6 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
for(int countEach_<%=cid%>: pstmt_<%=cid %>.executeBatch()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : <%=("INSERT").equals(dataAction)? "1" : "countEach_" +cid%>);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%dbLog.batch().executeDone(dbLog.str(dataAction));%>
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
@@ -224,7 +217,6 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
for(int countEach_<%=cid%>: e.getUpdateCounts()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : countEach_<%=cid%>);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
insertedCount_<%=cid%> += countSum_<%=cid%>;
|
||||
@@ -254,16 +246,14 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
resourceMap.put("statementClosed_<%=cid%>", true);
|
||||
<%
|
||||
if(!useExistingConnection){
|
||||
if(!("true").equals(useExistingConn)){
|
||||
if(!("").equals(commitEvery) && !("0").equals(commitEvery))
|
||||
{
|
||||
%>
|
||||
if (commitCounter_<%=cid%> > 0 && rowsToCommitCount_<%=cid%> != 0) {
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
if (commitCounter_<%=cid%> > 0) {
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
commitCounter_<%=cid%> = 0;
|
||||
}
|
||||
<%
|
||||
}
|
||||
|
||||
@@ -631,9 +631,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
@@ -711,9 +709,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
@@ -848,9 +844,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
@@ -1002,9 +996,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -1080,9 +1072,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmtInsert_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -1214,7 +1204,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
try {
|
||||
updateFlag_<%=cid%>=pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%>+updateFlag_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += updateFlag_<%=cid%>;
|
||||
|
||||
if(updateFlag_<%=cid%> == 0) {
|
||||
|
||||
<%
|
||||
@@ -1260,9 +1250,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmtInsert_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
}else{
|
||||
@@ -1354,9 +1342,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
deletedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
deletedCount_<%=cid%> = deletedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().deleting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -1424,9 +1410,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
%>
|
||||
int replaceCount_<%=cid%> = 0;
|
||||
try {
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
replaceCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
replaceCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().replacing(null);%>
|
||||
} catch(java.lang.Exception e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
@@ -1506,9 +1490,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
int count_on_duplicate_key_<%=cid%> = 0;
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
count_on_duplicate_key_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
count_on_duplicate_key_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().insertingOnDuplicateKeyUpdating(null);%>
|
||||
} catch(java.lang.Exception e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
@@ -1574,9 +1556,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().insertingOnDuplicateKeyUpdating(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
@@ -1647,7 +1627,6 @@ if(columnList != null && columnList.size() > 0) {
|
||||
for(int countEach_<%=cid%>: pstmt_<%=cid %>.executeBatch()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : 1);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%dbLog.batch().executeDone(dbLog.str(dataAction));%>
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
@@ -1666,7 +1645,6 @@ if(columnList != null && columnList.size() > 0) {
|
||||
for(int countEach_<%=cid%>: e.getUpdateCounts()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : countEach_<%=cid%>);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
insertedCount_<%=cid%> += countSum_<%=cid%>;
|
||||
@@ -1696,7 +1674,6 @@ if(columnList != null && columnList.size() > 0) {
|
||||
for(int countEach_<%=cid%>: pstmt_<%=cid %>.executeBatch()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : countEach_<%=cid%>);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%dbLog.batch().executeDone(dbLog.str(dataAction));%>
|
||||
<%
|
||||
if (("UPDATE").equals(dataAction)) {
|
||||
@@ -1720,7 +1697,6 @@ if(columnList != null && columnList.size() > 0) {
|
||||
for(int countEach_<%=cid%>: e.getUpdateCounts()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : countEach_<%=cid%>);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%if (("UPDATE").equals(dataAction)) {%>
|
||||
updatedCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<% }else if (("DELETE").equals(dataAction)) {%>
|
||||
@@ -1751,7 +1727,6 @@ if(columnList != null && columnList.size() > 0) {
|
||||
for(int countEach_<%=cid%>: pstmt_<%=cid %>.executeBatch()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : <%=("INSERT").equals(dataAction)? "1" : "countEach_" +cid%>);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%dbLog.batch().executeDone(dbLog.str(dataAction));%>
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
@@ -1778,7 +1753,6 @@ if(columnList != null && columnList.size() > 0) {
|
||||
for(int countEach_<%=cid%>: e.getUpdateCounts()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : countEach_<%=cid%>);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
insertedCount_<%=cid%> += countSum_<%=cid%>;
|
||||
@@ -1801,12 +1775,9 @@ if(columnList != null && columnList.size() > 0) {
|
||||
<%
|
||||
}
|
||||
%>
|
||||
if(rowsToCommitCount_<%=cid%> != 0){
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
}
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
commitCounter_<%=cid%>=0;
|
||||
}
|
||||
<%
|
||||
|
||||
@@ -188,7 +188,6 @@ int tmp_batchUpdateCount_<%=cid%> = 0;
|
||||
int deletedCount_<%=cid%>=0;
|
||||
int updatedCount_<%=cid%>=0;
|
||||
int insertedCount_<%=cid%>=0;
|
||||
int rowsToCommitCount_<%=cid%>=0;
|
||||
int rejectedCount_<%=cid%>=0;
|
||||
|
||||
boolean whetherReject_<%=cid%> = false;
|
||||
|
||||
@@ -5,7 +5,6 @@ imports="
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
org.talend.core.model.process.IConnection
|
||||
org.talend.core.model.utils.NodeUtil
|
||||
org.talend.core.model.process.IProcess
|
||||
java.util.List
|
||||
"
|
||||
%>
|
||||
@@ -16,7 +15,6 @@ imports="
|
||||
|
||||
String cid = node.getUniqueName();
|
||||
dbLog = new DBLogUtil(node);
|
||||
IProcess process = node.getProcess();
|
||||
String dataAction = ElementParameterParser.getValue(node,"__DATA_ACTION__");
|
||||
|
||||
String commitEvery = ElementParameterParser.getValue(node, "__COMMIT_EVERY__");
|
||||
@@ -24,7 +22,6 @@ imports="
|
||||
String dbVersion = ElementParameterParser.getValue(node, "__DB_VERSION__");
|
||||
|
||||
String connection = ElementParameterParser.getValue(node,"__CONNECTION__");
|
||||
boolean setAutoCommit = false;
|
||||
boolean useExistingConnection = "true".equals(ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__"));
|
||||
|
||||
String dieOnError = ElementParameterParser.getValue(node, "__DIE_ON_ERROR__");
|
||||
@@ -51,16 +48,6 @@ imports="
|
||||
IConnection rejectConn = rejectConns.get(0);
|
||||
rejectConnName = rejectConn.getName();
|
||||
}
|
||||
|
||||
if(useExistingConnection) {
|
||||
List<? extends INode> nodes = process.getNodesOfType("tAmazonOracleConnection");
|
||||
for (INode ne:nodes) {
|
||||
if (connection.equals(ne.getUniqueName())) {
|
||||
setAutoCommit = "true".equals(ElementParameterParser.getValue(ne, "__AUTO_COMMIT__"));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
boolean isUseBatchByDBVersion = (!useExistingConnection && !"ORACLE_8".equalsIgnoreCase(dbVersion)) || (useExistingConnection && !"ORACLE_8".equalsIgnoreCase(NodeUtil.getNodeByUniqueName(node.getProcess(), connection).getUniqueName()));
|
||||
|
||||
boolean useBatch = true;
|
||||
@@ -209,7 +196,6 @@ imports="
|
||||
<%
|
||||
}%>
|
||||
+= (tmp_batchUpdateCount_<%=cid%>!=-1?tmp_batchUpdateCount_<%=cid%>:0);
|
||||
rowsToCommitCount_<%=cid%> += tmp_batchUpdateCount_<%=cid%> != -1 ? tmp_batchUpdateCount_<%=cid%> : 0;
|
||||
}
|
||||
}
|
||||
<% } %>
|
||||
@@ -227,12 +213,10 @@ imports="
|
||||
if(!("").equals(commitEvery) && !("0").equals(commitEvery))
|
||||
{
|
||||
%>
|
||||
if (commitCounter_<%=cid%> > 0 && rowsToCommitCount_<%=cid%> != 0) {
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
if (commitCounter_<%=cid%> > 0 ) {
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
commitCounter_<%=cid%> = 0;
|
||||
}
|
||||
<%
|
||||
}
|
||||
|
||||
@@ -454,9 +454,7 @@ IMetadataColumn talendDynCol = null;
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
@@ -598,9 +596,7 @@ IMetadataColumn talendDynCol = null;
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
@@ -765,9 +761,7 @@ IMetadataColumn talendDynCol = null;
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -848,9 +842,7 @@ IMetadataColumn talendDynCol = null;
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmtInsert_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -988,7 +980,6 @@ IMetadataColumn talendDynCol = null;
|
||||
try {
|
||||
updateFlag_<%=cid%> = pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%>+updateFlag_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += updateFlag_<%=cid%>;
|
||||
if(updateFlag_<%=cid%> == 0) {
|
||||
|
||||
<%
|
||||
@@ -1037,9 +1028,7 @@ IMetadataColumn talendDynCol = null;
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmtInsert_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
}else{
|
||||
@@ -1134,9 +1123,7 @@ IMetadataColumn talendDynCol = null;
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
deletedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
deletedCount_<%=cid%> = deletedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().deleting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -1235,7 +1222,6 @@ IMetadataColumn talendDynCol = null;
|
||||
<%
|
||||
}%>
|
||||
+= (tmp_batchUpdateCount_<%=cid%>!=-1?tmp_batchUpdateCount_<%=cid%>:0);
|
||||
rowsToCommitCount_<%=cid%> += tmp_batchUpdateCount_<%=cid%> != -1 ? tmp_batchUpdateCount_<%=cid%> : 0;
|
||||
batchSizeCounter_<%=cid%> = 0;
|
||||
}
|
||||
<%
|
||||
@@ -1285,19 +1271,15 @@ IMetadataColumn talendDynCol = null;
|
||||
<%
|
||||
}%>
|
||||
+= (tmp_batchUpdateCount_<%=cid%>!=-1?tmp_batchUpdateCount_<%=cid%>:0);
|
||||
rowsToCommitCount_<%=cid%> += tmp_batchUpdateCount_<%=cid%> != -1 ? tmp_batchUpdateCount_<%=cid%> : 0;
|
||||
batchSizeCounter_<%=cid%> = 0;
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
if(rowsToCommitCount_<%=cid%> != 0){
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
}
|
||||
commitCounter_<%=cid%>=0;
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
commitCounter_<%=cid%>=0;
|
||||
}
|
||||
<%
|
||||
}
|
||||
|
||||
@@ -24,9 +24,6 @@
|
||||
String user = ElementParameterParser.getValue(node, "__USER__");
|
||||
String node_type = ElementParameterParser.getValue(node, "__NODE_TYPE__");
|
||||
String node_count = ElementParameterParser.getValue(node, "__NODE_COUNT__");
|
||||
if (node_count.isEmpty()) {
|
||||
node_count = "1";
|
||||
}
|
||||
|
||||
boolean isLog4jEnabled = "true".equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
|
||||
@@ -91,46 +88,52 @@
|
||||
.withMasterUsername(<%=user%>)
|
||||
.withMasterUserPassword(decryptedPwd_<%=cid%>)
|
||||
|
||||
.withNodeType(<%=node_type%>);
|
||||
.withNodeType(<%=node_type%>)
|
||||
|
||||
if (<%=node_count%> > 1) {
|
||||
request_<%=cid%> = request_<%=cid%>.withNumberOfNodes(<%=node_count%>);
|
||||
} else {
|
||||
request_<%=cid%> = request_<%=cid%>.withClusterType("single-node");
|
||||
}
|
||||
|
||||
//advanced settings
|
||||
<%if(parameterUtil.isValid(parameter_group_name)) {%>
|
||||
request_<%=cid%> = request_<%=cid%>.withClusterParameterGroupName(<%=parameter_group_name%>);
|
||||
<%}%>
|
||||
|
||||
<%if(parameterUtil.isValid(subnet_group_name)) {%>
|
||||
request_<%=cid%> = request_<%=cid%>.withClusterSubnetGroupName(<%=subnet_group_name%>);
|
||||
<%}%>
|
||||
|
||||
<%
|
||||
if(publicly_accessible) {
|
||||
%>
|
||||
request_<%=cid%> = request_<%=cid%>.withPubliclyAccessible(true);
|
||||
<%if(set_public_ip_address) {%>
|
||||
request_<%=cid%> = request_<%=cid%>.withElasticIp(<%=elastic_ip%>);
|
||||
<%
|
||||
if(new Integer(node_count) > 1){
|
||||
%>
|
||||
.withNumberOfNodes(<%=node_count%>)
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
.withClusterType("single-node")
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
//advanced settings
|
||||
<%if(parameterUtil.isValid(parameter_group_name)) {%>
|
||||
.withClusterParameterGroupName(<%=parameter_group_name%>)
|
||||
<%}%>
|
||||
|
||||
<%if(parameterUtil.isValid(subnet_group_name)) {%>
|
||||
.withClusterSubnetGroupName(<%=subnet_group_name%>)
|
||||
<%}%>
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
request_<%=cid%> = request_<%=cid%>.withPubliclyAccessible(false);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
<%if(parameterUtil.isValid(availability_zone)) {%>
|
||||
request_<%=cid%> = request_<%=cid%>.withAvailabilityZone(<%=availability_zone%>);
|
||||
<%}%>
|
||||
|
||||
<%if(parameterUtil.isValid(vpc_security_groupids)) {%>
|
||||
request_<%=cid%> = request_<%=cid%>.withVpcSecurityGroupIds(<%=vpc_security_groupids%>.split(","));
|
||||
<%}%>
|
||||
|
||||
|
||||
<%
|
||||
if(publicly_accessible) {
|
||||
%>
|
||||
.withPubliclyAccessible(true)
|
||||
<%if(set_public_ip_address) {%>
|
||||
.withElasticIp(<%=elastic_ip%>)
|
||||
<%}%>
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
.withPubliclyAccessible(false)
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
<%if(parameterUtil.isValid(availability_zone)) {%>
|
||||
.withAvailabilityZone(<%=availability_zone%>)
|
||||
<%}%>
|
||||
|
||||
<%if(parameterUtil.isValid(vpc_security_groupids)) {%>
|
||||
.withVpcSecurityGroupIds(<%=vpc_security_groupids%>.split(","))
|
||||
<%}%>
|
||||
;
|
||||
|
||||
com.amazonaws.services.redshift.model.Cluster result_<%=cid%> = client_<%=cid%>.createCluster(request_<%=cid%>);
|
||||
<%if(isLog4jEnabled) {%>
|
||||
|
||||
@@ -94,7 +94,6 @@ int nb_line_rejected_<%=cid%> = 0;
|
||||
int deletedCount_<%=cid%>=0;
|
||||
int updatedCount_<%=cid%>=0;
|
||||
int insertedCount_<%=cid%>=0;
|
||||
int rowsToCommitCount_<%=cid%>=0;
|
||||
int rejectedCount_<%=cid%>=0;
|
||||
String dbschema_<%=cid%> = null;
|
||||
String tableName_<%=cid%> = null;
|
||||
|
||||
@@ -5,10 +5,9 @@ imports="
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
org.talend.core.model.metadata.IMetadataTable
|
||||
org.talend.core.model.metadata.types.JavaTypesManager
|
||||
org.talend.core.model.metadata.MappingTypeRetriever
|
||||
org.talend.core.model.metadata.MetadataTalendType
|
||||
org.talend.core.model.metadata.MappingTypeRetriever
|
||||
org.talend.core.model.metadata.MetadataTalendType
|
||||
org.talend.core.model.process.IConnection
|
||||
org.talend.core.model.process.IProcess
|
||||
java.util.List
|
||||
java.util.ArrayList
|
||||
java.util.Map
|
||||
@@ -23,7 +22,6 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
|
||||
String cid = node.getUniqueName();
|
||||
dbLog = new DBLogUtil(node);
|
||||
IProcess process = node.getProcess();
|
||||
final String dbmsId = ElementParameterParser.getValue(node,"__MAPPING__");
|
||||
|
||||
List<Map<String, String>> addCols =
|
||||
@@ -34,12 +32,12 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
|
||||
String commitEvery = ElementParameterParser.getValue(node, "__COMMIT_EVERY__");
|
||||
|
||||
boolean setAutoCommit = false;
|
||||
|
||||
boolean useExistingConnection = "true".equals(ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__"));
|
||||
String useExistingConn = ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__");
|
||||
|
||||
boolean identityInsert= "true".equals(ElementParameterParser.getValue(node, "__IDENTITY_INSERT__"));
|
||||
|
||||
boolean useExistingConnection = "true".equals(ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__"));
|
||||
|
||||
boolean ignoreDateOutofRange = ("true").equals(ElementParameterParser.getValue(node,"__IGNORE_DATE_OUTOF_RANGE__"));
|
||||
boolean useFieldOptions = ("true").equals(ElementParameterParser.getValue(node, "__USE_FIELD_OPTIONS__"));
|
||||
List<Map<String, String>> fieldOptions = (List<Map<String,String>>)ElementParameterParser.getObjectValue(node, "__FIELD_OPTIONS__");
|
||||
@@ -75,17 +73,6 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
rejectConnName = rejectConn.getName();
|
||||
}
|
||||
|
||||
String connection = ElementParameterParser.getValue(node,"__CONNECTION__");
|
||||
if(useExistingConnection) {
|
||||
List<? extends INode> nodes = process.getNodesOfType("tAzureSynapseConnection");
|
||||
for (INode ne:nodes) {
|
||||
if (connection.equals(ne.getUniqueName())) {
|
||||
setAutoCommit = "true".equals(ElementParameterParser.getValue(ne, "__AUTO_COMMIT__"));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
boolean useBatch = useBatchSize && (rejectConnName == null)
|
||||
&& (
|
||||
("INSERT").equals(dataAction) || ("UPDATE").equals(dataAction) || ("DELETE").equals(dataAction)
|
||||
@@ -376,9 +363,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
|
||||
row2Count<%=cid%>++;
|
||||
}
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
}
|
||||
<% }else {
|
||||
%>
|
||||
@@ -395,7 +380,6 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
countSum_<%=cid%> += countEach_<%=cid%>;
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%dbLog.batch().executeDone(dbLog.str(dataAction));%>
|
||||
}
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
@@ -422,7 +406,6 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
for(int countEach_<%=cid%>: e.getUpdateCounts()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : countEach_<%=cid%>);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
insertedCount_<%=cid%> += countSum_<%=cid%>;
|
||||
@@ -474,18 +457,14 @@ if(identityInsert){
|
||||
%>
|
||||
resourceMap.put("statementClosed_<%=cid%>", true);
|
||||
<%
|
||||
if(!useExistingConnection)
|
||||
if(!("true").equals(useExistingConn))
|
||||
{
|
||||
if(!("").equals(commitEvery) && !("0").equals(commitEvery))
|
||||
{
|
||||
%>
|
||||
if(rowsToCommitCount_<%=cid%> != 0){
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
commitCounter_<%=cid%> = 0;
|
||||
}
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
<%
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user