Compare commits
311 Commits
patch/TPS-
...
release/7.
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
196bec65e5 | ||
|
|
c6fdf4da5b | ||
|
|
f52c9c2584 | ||
|
|
0fb3315eee | ||
|
|
ba12b6577a | ||
|
|
ce7bfa060f | ||
|
|
a8155b659c | ||
|
|
f0c4517bbc | ||
|
|
247fc84baa | ||
|
|
1b8fe81314 | ||
|
|
5d32586b46 | ||
|
|
4596f2a3a8 | ||
|
|
f5001bcddf | ||
|
|
15f87a595f | ||
|
|
74d0a5fc4d | ||
|
|
9a7f158ea8 | ||
|
|
76aff58dc9 | ||
|
|
ab1c2ac982 | ||
|
|
856a9343b8 | ||
|
|
1db0ec3664 | ||
|
|
b39ca1b28e | ||
|
|
54e445bdd2 | ||
|
|
9cbe98d4ef | ||
|
|
26e88b26be | ||
|
|
6217c3042d | ||
|
|
2c8df34e97 | ||
|
|
86b8e75e73 | ||
|
|
f2bcb6d71f | ||
|
|
0403d4d7d4 | ||
|
|
4198afd1d2 | ||
|
|
a971215151 | ||
|
|
14c59926c6 | ||
|
|
374903d266 | ||
|
|
abab035072 | ||
|
|
03c3e264e6 | ||
|
|
948eb33247 | ||
|
|
70b86c3519 | ||
|
|
57ef3ab3f6 | ||
|
|
bb0dc580a2 | ||
|
|
2f1029eba9 | ||
|
|
fff59f3b2f | ||
|
|
712db2045d | ||
|
|
96bc7ed4c5 | ||
|
|
f749ac3e2a | ||
|
|
91e83eedf0 | ||
|
|
4d5844376e | ||
|
|
e56ccf3f62 | ||
|
|
e04ef82e30 | ||
|
|
273a5025c7 | ||
|
|
612e5a5c27 | ||
|
|
38f544be0a | ||
|
|
3b872b62e5 | ||
|
|
1910ed40e5 | ||
|
|
d0912599ec | ||
|
|
ee20cf02ca | ||
|
|
5427bc53f1 | ||
|
|
9b4bfe85be | ||
|
|
b5916d9122 | ||
|
|
5e9f08c9ff | ||
|
|
8e977f5110 | ||
|
|
90c7f820b9 | ||
|
|
1d5f0bacc6 | ||
|
|
8a3ef5e9ad | ||
|
|
47bd697fbe | ||
|
|
926cb4f36b | ||
|
|
2ffffcc703 | ||
|
|
c010bc5084 | ||
|
|
3dc2aac9bb | ||
|
|
9f7ad6f9dd | ||
|
|
7426a93b49 | ||
|
|
c3e5d003be | ||
|
|
cf5ea4e526 | ||
|
|
4914a382fc | ||
|
|
30caab1c84 | ||
|
|
da989e3f8c | ||
|
|
77754871b5 | ||
|
|
b024dd0ec0 | ||
|
|
b4d3e5ebea | ||
|
|
29f74206d5 | ||
|
|
445c9fc74b | ||
|
|
9f0b02c045 | ||
|
|
e3f4be3219 | ||
|
|
dc47689710 | ||
|
|
936cf7958f | ||
|
|
8f842b2155 | ||
|
|
db23bf8ec1 | ||
|
|
99ba1911f3 | ||
|
|
24ef38e2da | ||
|
|
190acc37f8 | ||
|
|
c9dd09c733 | ||
|
|
263b021fca | ||
|
|
76fbb18558 | ||
|
|
d89f4d424f | ||
|
|
c8b397b4a6 | ||
|
|
bd0d7e0afd | ||
|
|
f36de6e55a | ||
|
|
a5a41af6cf | ||
|
|
c5b23dd28f | ||
|
|
fec8a84ca1 | ||
|
|
f77ff96de2 | ||
|
|
df4e255819 | ||
|
|
5cf8a97e87 | ||
|
|
792b1ccc01 | ||
|
|
6507179daa | ||
|
|
3627e9fc1f | ||
|
|
572f642911 | ||
|
|
e06da9671f | ||
|
|
dc1ec06b42 | ||
|
|
7dfac8dc4f | ||
|
|
f398c2d5f8 | ||
|
|
67a155faa3 | ||
|
|
8fe63a7afc | ||
|
|
52665b6347 | ||
|
|
1dfd7ea0ec | ||
|
|
7b35b37054 | ||
|
|
d99fd9c2e5 | ||
|
|
072795ea1b | ||
|
|
ceb555dc51 | ||
|
|
67fb0f4d12 | ||
|
|
47deaa8326 | ||
|
|
94128e4cbf | ||
|
|
935b6c3996 | ||
|
|
08b538ff57 | ||
|
|
033fd28354 | ||
|
|
00505282ac | ||
|
|
c33d8c379b | ||
|
|
64a0bf8012 | ||
|
|
195eaa06e7 | ||
|
|
d59aa3ca97 | ||
|
|
21c18c0374 | ||
|
|
4d5a9e18b8 | ||
|
|
d7b6cfecf2 | ||
|
|
08c045ee3a | ||
|
|
11cf523196 | ||
|
|
c5ce3c8df9 | ||
|
|
de51fcbd6b | ||
|
|
1e3fe2f3b6 | ||
|
|
48e6c26fa2 | ||
|
|
8908f7bb50 | ||
|
|
73a7c3d0f6 | ||
|
|
329a38171c | ||
|
|
7f292079bb | ||
|
|
e1ee2bd473 | ||
|
|
a2b1281156 | ||
|
|
83421a7f6c | ||
|
|
721258942c | ||
|
|
eb31686a3e | ||
|
|
223f5da7db | ||
|
|
a961d842ea | ||
|
|
922d138523 | ||
|
|
1c534b4b5f | ||
|
|
8dd9933294 | ||
|
|
09fd6c52ff | ||
|
|
0399e751f5 | ||
|
|
9ca8fa0663 | ||
|
|
43cb82c389 | ||
|
|
f3bfc9ca6b | ||
|
|
8a3ef23853 | ||
|
|
d2e9b874c1 | ||
|
|
e7e9908678 | ||
|
|
da0a057872 | ||
|
|
9052eb3e99 | ||
|
|
b88c6e3fbc | ||
|
|
457bd0fc40 | ||
|
|
c4c49001a2 | ||
|
|
c21f8e7551 | ||
|
|
cda47b2324 | ||
|
|
cab755152c | ||
|
|
14ed765ab8 | ||
|
|
1844d76b67 | ||
|
|
b3f4d37498 | ||
|
|
1c786fb4f2 | ||
|
|
d54077e7dc | ||
|
|
9d8ffa6d37 | ||
|
|
e5b9def018 | ||
|
|
08fb15d12b | ||
|
|
83d684ce0c | ||
|
|
2aa3688f12 | ||
|
|
229c1f8d95 | ||
|
|
c8325d6799 | ||
|
|
7139c9bb7f | ||
|
|
e73b0715dc | ||
|
|
ad56747c35 | ||
|
|
8faeb6dd45 | ||
|
|
706b61e22d | ||
|
|
fd891e2419 | ||
|
|
8599590f33 | ||
|
|
7c5a32acb1 | ||
|
|
600b37d142 | ||
|
|
c4900e7ad2 | ||
|
|
6d8adab44d | ||
|
|
751c6d7813 | ||
|
|
22792ee2a3 | ||
|
|
34b7458303 | ||
|
|
2f7a4e930d | ||
|
|
b3ea83c580 | ||
|
|
5920f991b0 | ||
|
|
1c40d8b173 | ||
|
|
4b56340f46 | ||
|
|
926bc55d86 | ||
|
|
e27ab0d505 | ||
|
|
4b1dd0a0c5 | ||
|
|
8ed935fc5f | ||
|
|
18b2039453 | ||
|
|
31d5836c01 | ||
|
|
6e88f43352 | ||
|
|
efd512cc4f | ||
|
|
5632263a99 | ||
|
|
b8d0a79ae6 | ||
|
|
1a1d07909f | ||
|
|
c9c865871b | ||
|
|
4eb95ff314 | ||
|
|
9058e5f0cf | ||
|
|
9db887667a | ||
|
|
115ff04f57 | ||
|
|
12181f8a75 | ||
|
|
ccdad6daa8 | ||
|
|
62dc6af821 | ||
|
|
739ca0b32c | ||
|
|
534feea221 | ||
|
|
f4a45b3659 | ||
|
|
e3a55d8b63 | ||
|
|
80fbfd17a8 | ||
|
|
0652420c79 | ||
|
|
23e645e8e6 | ||
|
|
2dcba4cddb | ||
|
|
9e7e4c0971 | ||
|
|
d5f863477d | ||
|
|
b802bb6a72 | ||
|
|
21b8a407de | ||
|
|
0352e0556c | ||
|
|
9ce74b5a90 | ||
|
|
b9c4102074 | ||
|
|
2733478a43 | ||
|
|
9b2180562c | ||
|
|
bec10957e5 | ||
|
|
d0a5929ad6 | ||
|
|
cc999c45b5 | ||
|
|
5defbea32b | ||
|
|
2583507d00 | ||
|
|
48b0e2ecbd | ||
|
|
f1b204b226 | ||
|
|
7f5e8bfa11 | ||
|
|
063f1daeec | ||
|
|
d2621d049e | ||
|
|
364d240804 | ||
|
|
7398c13e69 | ||
|
|
ea4ae4c735 | ||
|
|
5eb3f438ea | ||
|
|
dc1c743d48 | ||
|
|
b885667b38 | ||
|
|
4019138048 | ||
|
|
eb00028926 | ||
|
|
f439623eab | ||
|
|
8e65ae765f | ||
|
|
84edd12af5 | ||
|
|
cb8308df05 | ||
|
|
62fe9a0f38 | ||
|
|
ac80911050 | ||
|
|
7c682778ec | ||
|
|
4159e31ef1 | ||
|
|
b7ed047c91 | ||
|
|
be5a9a7dba | ||
|
|
0f0ead31aa | ||
|
|
70f45909b9 | ||
|
|
3e0d0538bb | ||
|
|
40d9315e48 | ||
|
|
0946243005 | ||
|
|
d544cfbbd8 | ||
|
|
38d73d1c3b | ||
|
|
a257812d17 | ||
|
|
4224c49747 | ||
|
|
2a48cfd3c5 | ||
|
|
c4adf887db | ||
|
|
18422df2a4 | ||
|
|
0985646d21 | ||
|
|
481edbbc18 | ||
|
|
02df23ffeb | ||
|
|
957704c0a4 | ||
|
|
cb85a154f8 | ||
|
|
c7c9d4bf2d | ||
|
|
4271923ca3 | ||
|
|
70f0e7a076 | ||
|
|
6e0af4dd99 | ||
|
|
26fbb4e014 | ||
|
|
28b472384b | ||
|
|
c0f93ad66b | ||
|
|
7a53a97ceb | ||
|
|
1c89042c03 | ||
|
|
9133ff39d1 | ||
|
|
1d706bc243 | ||
|
|
24031c6bb0 | ||
|
|
5d7deb4f08 | ||
|
|
2f323ea5ec | ||
|
|
0099cf4b7f | ||
|
|
ad8b244f13 | ||
|
|
b4196f3ade | ||
|
|
a6cc1fd725 | ||
|
|
6d2c08fb02 | ||
|
|
b37f4d2169 | ||
|
|
18a6bd6b9e | ||
|
|
12025462fd | ||
|
|
8a17273fff | ||
|
|
4666ad0ba2 | ||
|
|
e99e3a88ac | ||
|
|
eef7ee7939 | ||
|
|
323b7fdf86 | ||
|
|
9e8d8785c2 | ||
|
|
463d35acb2 | ||
|
|
5069b086d7 | ||
|
|
18694afab5 |
@@ -55,5 +55,6 @@
|
||||
<plugin id="org.talend.testutils" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
|
||||
<plugin id="org.talend.updates.runtime.test" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
|
||||
<plugin id="org.talend.utils.test" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
|
||||
<plugin id="org.talend.sdk.component.studio-integration.test" download-size="0" install-size="0" version="0.0.0" fragment="true" unpack="false"/>
|
||||
<plugin id="test.all.test.suite" download-size="0" install-size="0" version="0.0.0" unpack="false"/>
|
||||
</feature>
|
||||
|
||||
@@ -22,6 +22,7 @@ Require-Bundle: org.talend.core.repository,
|
||||
org.apache.log4j,
|
||||
org.talend.components.api,
|
||||
org.talend.daikon,
|
||||
org.talend.daikon.crypto.utils,
|
||||
org.talend.designer.core.generic
|
||||
Eclipse-LazyStart: true
|
||||
Bundle-ClassPath: .
|
||||
|
||||
@@ -59,88 +59,76 @@ if((codePart.equals(ECodePart.END))&&(stat || logstashCurrent)){
|
||||
List<String> needToEndConnNames = new ArrayList<String>();
|
||||
INode nextNode = node.getOutgoingConnections(EConnectionType.ITERATE).get(0).getTarget();
|
||||
NodeUtil.fillConnectionsForStat(needToEndConnNames, nextNode);
|
||||
%>
|
||||
|
||||
<%if(stat) {%>
|
||||
if(execStat){
|
||||
<%
|
||||
for(String connName : needToEndConnNames){
|
||||
if(!needToEndConnNames.isEmpty()) {
|
||||
if(stat && logstashCurrent) {
|
||||
%>
|
||||
runStat.updateStatOnConnection("<%=connName%>"+iterateId,2, 0);
|
||||
runStat.updateStatAndLog(execStat,enableLogStash,iterateId,2,0<%for(String connName : needToEndConnNames){%>,"<%=connName%>"<%}%>);
|
||||
<%
|
||||
} else {
|
||||
if(stat) {%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection(iterateId,2,0<%for(String connName : needToEndConnNames){%>,"<%=connName%>"<%}%>);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {//now only finish the log, not send, TODO%>
|
||||
if(enableLogStash){
|
||||
runStat.log(iterateId,2,0<%for(String connName : needToEndConnNames){%>,"<%=connName%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {//now only finish the log, not send, TODO%>
|
||||
if(enableLogStash){
|
||||
<%
|
||||
for(String connName : needToEndConnNames){
|
||||
%>
|
||||
runStat.logStatOnConnection("<%=connName%>"+iterateId,2, 0);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%
|
||||
}
|
||||
if(connSet.size()>0){
|
||||
for(IConnection con:connSet){
|
||||
INode source = con.getSource();
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
%>
|
||||
<%if(stat) {%>
|
||||
if(execStat){
|
||||
if(resourceMap.get("inIterateVComp") == null || !((Boolean)resourceMap.get("inIterateVComp"))){
|
||||
runStat.updateStatOnConnection("<%=con.getUniqueName()%>"+iterateId,2, 0);
|
||||
}
|
||||
|
||||
if(connSet.size()>0) {
|
||||
if(stat && logstashCurrent && (connSet.size()==1)) {//the most common case, write this ugly logic for 65535 issue
|
||||
for(IConnection con:connSet){
|
||||
INode source = con.getSource();
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
%>
|
||||
if(runStat.updateStatAndLog(execStat,enableLogStash,resourceMap,iterateId,"<%=con.getUniqueName()%>",2,0,
|
||||
<%=jobStructureCatcher.getUniqueName()%>,"<%=sourceNodeId%>","<%=sourceNodeComponent%>","<%=node.getUniqueName()%>","<%=node.getComponent().getName()%>","<%="REJECT".equals(con.getConnectorName()) ? "reject" : "output"%>")) {
|
||||
<%=jobStructureCatcher.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
}
|
||||
<%
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if(stat){
|
||||
%>
|
||||
if(execStat){
|
||||
runStat.updateStat(resourceMap,iterateId,2,0<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash){
|
||||
if(resourceMap.get("inIterateVComp") == null || !((Boolean)resourceMap.get("inIterateVComp"))){
|
||||
<%
|
||||
if(logstashCurrent){
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
<%
|
||||
for(IConnection con:connSet){
|
||||
INode source = con.getSource();
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
%>
|
||||
RunStat.StatBean talend_statebean = runStat.logStatOnConnection("<%=con.getUniqueName()%>"+iterateId,2, 0);
|
||||
|
||||
<%=jobStructureCatcher.getUniqueName() %>.addConnectionMessage(
|
||||
"<%=sourceNodeId%>",
|
||||
"<%=sourceNodeComponent%>",
|
||||
false,
|
||||
"<%="REJECT".equals(con.getConnectorName()) ? "reject" : "output"%>",
|
||||
"<%=con.getUniqueName()%>",
|
||||
talend_statebean.getNbLine(),
|
||||
talend_statebean.getStartTime(),
|
||||
talend_statebean.getEndTime()
|
||||
);
|
||||
|
||||
<%=jobStructureCatcher.getUniqueName() %>.addConnectionMessage(
|
||||
"<%=node.getUniqueName()%>",
|
||||
"<%=node.getComponent().getName()%>",
|
||||
true,
|
||||
"input",
|
||||
"<%=con.getUniqueName()%>",
|
||||
talend_statebean.getNbLine(),
|
||||
talend_statebean.getStartTime(),
|
||||
talend_statebean.getEndTime()
|
||||
);
|
||||
<%=jobStructureCatcher.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
if(runStat.log(resourceMap,iterateId,"<%=con.getUniqueName()%>",2,0,
|
||||
<%=jobStructureCatcher.getUniqueName()%>,"<%=sourceNodeId%>","<%=sourceNodeComponent%>","<%=node.getUniqueName()%>","<%=node.getComponent().getName()%>","<%="REJECT".equals(con.getConnectorName()) ? "reject" : "output"%>")) {
|
||||
<%=jobStructureCatcher.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
}
|
||||
<%
|
||||
break;
|
||||
}
|
||||
%>
|
||||
}
|
||||
}
|
||||
%>
|
||||
}
|
||||
<%
|
||||
}
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
@@ -341,14 +329,26 @@ end_Hash.put("<%=node.getUniqueName() %>", System.currentTimeMillis());
|
||||
}
|
||||
// Else, that means the component is a ON_COMPONENT_OK virtual component typed.
|
||||
}
|
||||
|
||||
|
||||
if(generateMethodCall) {
|
||||
/*check if parralel iterate call not finished from this component */
|
||||
boolean parallelIterate = false;
|
||||
for (IConnection iterateConn : iterateConnSet) {
|
||||
parallelIterate = "true".equals(ElementParameterParser.getValue(iterateConn, "__ENABLE_PARALLEL__"));
|
||||
if (parallelIterate) {
|
||||
if (codePart.equals(ECodePart.END)) {
|
||||
String iterateTargetNodeName = iterateConn.getTarget().getUniqueName();
|
||||
%>
|
||||
mtp_<%=iterateTargetNodeName %>.waitForEndOfQueue();
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
<%=outgoingConn.getTarget().getUniqueName() %>Process(globalMap);
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (outgoingConn.getLineStyle().equals(EConnectionType.RUN_IF)) {
|
||||
%>
|
||||
@@ -356,12 +356,27 @@ end_Hash.put("<%=node.getUniqueName() %>", System.currentTimeMillis());
|
||||
<%//send the true status to socket
|
||||
if(stat){
|
||||
%>
|
||||
if(execStat){
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection("<%=outgoingConn.getUniqueName() %>", 0, "true");
|
||||
}
|
||||
<%}%>
|
||||
<%=outgoingConn.getTarget().getUniqueName() %>Process(globalMap);
|
||||
}
|
||||
<%
|
||||
}
|
||||
/*check if parralel iterate call not finished from this component */
|
||||
boolean parallelIterate = false;
|
||||
for (IConnection iterateConn : iterateConnSet) {
|
||||
parallelIterate = "true".equals(ElementParameterParser.getValue(iterateConn, "__ENABLE_PARALLEL__"));
|
||||
if (parallelIterate) {
|
||||
if (codePart.equals(ECodePart.END)) {
|
||||
String iterateTargetNodeName = iterateConn.getTarget().getUniqueName();
|
||||
%>
|
||||
mtp_<%=iterateTargetNodeName %>.waitForEndOfQueue();
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
<%=outgoingConn.getTarget().getUniqueName() %>Process(globalMap);
|
||||
}
|
||||
|
||||
<%
|
||||
//send the false status to socket
|
||||
@@ -388,7 +403,7 @@ end_Hash.put("<%=node.getUniqueName() %>", System.currentTimeMillis());
|
||||
parallelIterate = "true".equals(ElementParameterParser.getValue(iterateConn, "__ENABLE_PARALLEL__"));
|
||||
if (parallelIterate) {
|
||||
if (codePart.equals(ECodePart.END)) {
|
||||
String iterateTargetNodeName = iterateConn.getTarget().getUniqueName();
|
||||
String iterateTargetNodeName = iterateConn.getTarget().getUniqueName();
|
||||
%>
|
||||
mtp_<%=iterateTargetNodeName %>.waitForEndOfQueue();
|
||||
|
||||
|
||||
@@ -146,83 +146,82 @@
|
||||
boolean logstashCurrent = !cid.startsWith("tJobStructureCatcher") && !cid.startsWith("talend") && enableLogStash;
|
||||
|
||||
if ((codePart.equals(ECodePart.BEGIN))&&(stat || logstashCurrent)&&connSet.size()>0) {
|
||||
for(IConnection con:connSet){
|
||||
%>
|
||||
if (<%if(stat){%>execStat<%}%><%if(stat && logstashCurrent){%> || <%}%><%if(logstashCurrent){%>enableLogStash<%}%>) {
|
||||
if(resourceMap.get("inIterateVComp") == null){
|
||||
<%if(containsTPartitioner){%>
|
||||
java.util.concurrent.ConcurrentHashMap<Object, Object> concurrentHashMap_<%=con.getUniqueName() %> = (java.util.concurrent.ConcurrentHashMap) globalMap.get("concurrentHashMap");
|
||||
concurrentHashMap_<%=con.getUniqueName() %>.putIfAbsent("<%=con.getUniqueName() %>" + iterateLoop,new java.util.concurrent.atomic.AtomicInteger(0));
|
||||
java.util.concurrent.atomic.AtomicInteger stats_<%=con.getUniqueName() %> = (java.util.concurrent.atomic.AtomicInteger) concurrentHashMap_<%=con.getUniqueName() %>.get("<%=con.getUniqueName() %>" + iterateLoop);
|
||||
int step_<%=con.getUniqueName() %> = stats_<%=con.getUniqueName() %>.incrementAndGet()<=1?0:1;
|
||||
<%if(stat) {%>
|
||||
if(execStat) {
|
||||
runStat.updateStatOnConnection("<%=con.getUniqueName() %>"+iterateId, step_<%=con.getUniqueName()%>, 0);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.logStatOnConnection("<%=con.getUniqueName()%>"+iterateId, step_<%=con.getUniqueName()%>, 0);
|
||||
}
|
||||
<%}%>
|
||||
<%}else{%>
|
||||
<%if(stat) {%>
|
||||
if(execStat) {
|
||||
runStat.updateStatOnConnection("<%=con.getUniqueName() %>"+iterateId, 0, 0);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.logStatOnConnection("<%=con.getUniqueName()%>"+iterateId, 0, 0);
|
||||
}
|
||||
<%}%>
|
||||
<%}%>
|
||||
if(containsTPartitioner) {
|
||||
%>
|
||||
if(<%if(stat){%>execStat<%}%><%if(stat && logstashCurrent){%> || <%}%><%if(logstashCurrent){%>enableLogStash<%}%>) {
|
||||
runStat.updateStatOnConnectionAndLog(resourceMap,globalMap,iterateLoop,iterateId,<%if(stat){%>execStat<%} else {%>false<%}%>,enableLogStash,0<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
if(stat && logstashCurrent) {
|
||||
%>
|
||||
runStat.updateStatAndLog(execStat,enableLogStash,resourceMap,iterateId,0,0<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
<%
|
||||
} else {
|
||||
if(stat) {
|
||||
%>
|
||||
if(execStat) {
|
||||
runStat.updateStatOnConnection(resourceMap,iterateId,0,0<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
<%
|
||||
|
||||
if(logstashCurrent) {
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
runStat.log(resourceMap,iterateId,0,0<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if((codePart.equals(ECodePart.MAIN))&&(stat || logstashCurrent)&&connSet.size()>0){
|
||||
for(IConnection con:connSet){
|
||||
%>
|
||||
|
||||
//<%=con.getUniqueName()%>
|
||||
//<%=(String)codeGenArgument.getIncomingName()%>
|
||||
|
||||
|
||||
<%if (!node.getComponent().useMerge()) {%>
|
||||
<%if(stat) {%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection("<%=con.getUniqueName() %>"+iterateId,1, 1);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.logStatOnConnection("<%=con.getUniqueName() %>"+iterateId,1, 1);
|
||||
}
|
||||
<%}%>
|
||||
<%
|
||||
} else if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){
|
||||
if(!node.getComponent().useMerge()) {
|
||||
if(stat && logstashCurrent) {
|
||||
%>
|
||||
<%if(stat) {%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection("<%=con.getUniqueName() %>"+iterateId,1, 1);
|
||||
runStat.updateStatAndLog(execStat,enableLogStash,iterateId,1,1<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
<%
|
||||
} else {
|
||||
if(stat) {
|
||||
%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection(iterateId,1,1<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.logStatOnConnection("<%=con.getUniqueName() %>"+iterateId,1, 1);
|
||||
if(logstashCurrent) {
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
runStat.log(iterateId,1,1<%for(IConnection con : connSet){%>,"<%=con.getUniqueName()%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
<%}%>
|
||||
<%}%>
|
||||
|
||||
<%
|
||||
}
|
||||
} else {
|
||||
for(IConnection connection:connSet){
|
||||
if(connection.getUniqueName().equals((String)codeGenArgument.getIncomingName())){
|
||||
if(stat && logstashCurrent) {
|
||||
%>
|
||||
runStat.updateStatAndLog(execStat,enableLogStash,iterateId,1,1<%for(IConnection con : connSet){if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){%>,"<%=con.getUniqueName()%>"<%}}%>);
|
||||
<%
|
||||
} else {
|
||||
if(stat) {%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection(iterateId,1,1<%for(IConnection con : connSet){if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){%>,"<%=con.getUniqueName()%>"<%}}%>);
|
||||
}
|
||||
<%}
|
||||
|
||||
if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.log(iterateId,1,1<%for(IConnection con : connSet){if(con.getUniqueName().equals((String)codeGenArgument.getIncomingName())){%>,"<%=con.getUniqueName()%>"<%}}%>);
|
||||
}
|
||||
<%}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,7 +250,7 @@
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
<%=jobStructureCatcher.getUniqueName() %>.addComponentMessage("<%=node.getUniqueName()%>", "<%=node.getComponent().getName()%>");
|
||||
<%=jobStructureCatcher.getUniqueName() %>.addCM("<%=node.getUniqueName()%>", "<%=node.getComponent().getName()%>");
|
||||
<%=jobStructureCatcher.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
}
|
||||
<%
|
||||
@@ -266,46 +265,37 @@
|
||||
List<String> needToStartConnNames = new ArrayList<String>();
|
||||
INode nextNode = node.getOutgoingConnections(EConnectionType.ITERATE).get(0).getTarget();
|
||||
NodeUtil.fillConnectionsForStat(needToStartConnNames, nextNode);
|
||||
if(needToStartConnNames.isEmpty()) {
|
||||
//do nothing
|
||||
} else if(containsTPartitioner){
|
||||
%>
|
||||
if(<%if(stat){%>execStat<%}%><%if(stat && logstashCurrent){%> || <%}%><%if(logstashCurrent){%>enableLogStash<%}%>){
|
||||
<%
|
||||
for(String connName : needToStartConnNames){
|
||||
%>
|
||||
<%if(containsTPartitioner){%>
|
||||
java.util.concurrent.ConcurrentHashMap<Object, Object> concurrentHashMap_<%=connName%> = (java.util.concurrent.ConcurrentHashMap) globalMap.get("concurrentHashMap");
|
||||
concurrentHashMap_<%=connName%>.putIfAbsent("<%=connName%>" + iterateLoop,new java.util.concurrent.atomic.AtomicInteger(0));
|
||||
java.util.concurrent.atomic.AtomicInteger stats_<%=connName%> = (java.util.concurrent.atomic.AtomicInteger) concurrentHashMap_<%=connName%>.get("<%=connName%>" + iterateLoop);
|
||||
|
||||
int step_<%=connName %> = stats_<%=connName%>.incrementAndGet()<=1?0:1;
|
||||
<%if(stat) {%>
|
||||
if(execStat) {
|
||||
runStat.updateStatOnConnection("<%=connName%>"+iterateId, step_<%=connName%>, 0);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.logStatOnConnection("<%=connName%>"+iterateId, step_<%=connName%>, 0);
|
||||
}
|
||||
<%}%>
|
||||
<%}else{%>
|
||||
<%if(stat) {%>
|
||||
if(execStat) {
|
||||
runStat.updateStatOnConnection("<%=connName%>"+iterateId, 0, 0);
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash) {
|
||||
runStat.logStatOnConnection("<%=connName%>"+iterateId, 0, 0);
|
||||
}
|
||||
<%}%>
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
%>
|
||||
runStat.updateStatOnConnectionAndLog(globalMap,iterateLoop,iterateId,<%if(stat){%>execStat<%} else {%>false<%}%>,enableLogStash,0<%for(String connName : needToStartConnNames){%>,"<%=connName%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
if(stat && logstashCurrent) {
|
||||
%>
|
||||
runStat.updateStatAndLog(execStat,enableLogStash,iterateId,0,0<%for(String connName : needToStartConnNames){%>,"<%=connName%>"<%}%>);
|
||||
<%
|
||||
} else {
|
||||
if(stat){
|
||||
%>
|
||||
if(execStat){
|
||||
runStat.updateStatOnConnection(iterateId,0,0<%for(String connName : needToStartConnNames){%>,"<%=connName%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
<%if(logstashCurrent) {%>
|
||||
if(enableLogStash){
|
||||
runStat.log(iterateId,0,0<%for(String connName : needToStartConnNames){%>,"<%=connName%>"<%}%>);
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
<%
|
||||
}else if(codePart.equals(ECodePart.MAIN)){
|
||||
%>
|
||||
resourceMap.put("inIterateVComp", true);
|
||||
|
||||
@@ -54,7 +54,9 @@
|
||||
jobFolderName = jobFolderName + JavaResourcesHelper.getJobFolderName(process.getName(), process.getVersion());
|
||||
String jobClassPackageFolder = codeGenArgument.getCurrentProjectName().toLowerCase() + '/' + jobFolderName;
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(process, "__LOG4J_ACTIVATE__"));
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(process, "__LOG4J_ACTIVATE__"));//log4j enable
|
||||
boolean isLog4j2Enabled = ("true").equals(ElementParameterParser.getValue(process, "__LOG4J2_ACTIVATE__"));//log4j2 enable
|
||||
boolean isLog4j1Enabled = !isLog4j2Enabled;//log4j1 enable
|
||||
|
||||
boolean exist_tParallelize = false;
|
||||
List<? extends INode> tParallelizeList = process.getNodesOfType("tParallelize");
|
||||
@@ -445,9 +447,11 @@
|
||||
lastStr = "";
|
||||
}
|
||||
}
|
||||
enableLogStash = "true".equalsIgnoreCase(System.getProperty("monitoring"));
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
if(!"".equals(log4jLevel)){
|
||||
<%if(isLog4j1Enabled){%>
|
||||
if("trace".equalsIgnoreCase(log4jLevel)){
|
||||
log.setLevel(org.apache.log4j.Level.TRACE);
|
||||
}else if("debug".equalsIgnoreCase(log4jLevel)){
|
||||
@@ -464,6 +468,26 @@
|
||||
log.setLevel(org.apache.log4j.Level.OFF);
|
||||
}
|
||||
org.apache.log4j.Logger.getRootLogger().setLevel(log.getLevel());
|
||||
<%}%>
|
||||
|
||||
<%if(isLog4j2Enabled){%>
|
||||
if("trace".equalsIgnoreCase(log4jLevel)){
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel(log.getName(), org.apache.logging.log4j.Level.TRACE);
|
||||
}else if("debug".equalsIgnoreCase(log4jLevel)){
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel(log.getName(), org.apache.logging.log4j.Level.DEBUG);
|
||||
}else if("info".equalsIgnoreCase(log4jLevel)){
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel(log.getName(), org.apache.logging.log4j.Level.INFO);
|
||||
}else if("warn".equalsIgnoreCase(log4jLevel)){
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel(log.getName(), org.apache.logging.log4j.Level.WARN);
|
||||
}else if("error".equalsIgnoreCase(log4jLevel)){
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel(log.getName(), org.apache.logging.log4j.Level.ERROR);
|
||||
}else if("fatal".equalsIgnoreCase(log4jLevel)){
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel(log.getName(), org.apache.logging.log4j.Level.FATAL);
|
||||
}else if ("off".equalsIgnoreCase(log4jLevel)){
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel(log.getName(), org.apache.logging.log4j.Level.OFF);
|
||||
}
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel(org.apache.logging.log4j.LogManager.getRootLogger().getName(), log.getLevel());
|
||||
<%}%>
|
||||
}
|
||||
log.info("TalendJob: '<%=codeGenArgument.getJobName()%>' - Start.");
|
||||
<%}%>
|
||||
@@ -493,7 +517,18 @@
|
||||
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("appender.file.maxsize", "52428800");
|
||||
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("appender.file.maxbackup", "20");
|
||||
properties_<%=jobCatcherNode.getUniqueName()%>.setProperty("host", "false");
|
||||
|
||||
System.getProperties().stringPropertyNames().stream()
|
||||
.filter(it -> it.startsWith("monitoring.audit.logger.properties."))
|
||||
.forEach(key -> properties_<%=jobCatcherNode.getUniqueName()%>.setProperty(key.substring("monitoring.audit.logger.properties.".length()), System.getProperty(key)));
|
||||
|
||||
<%if(isLog4j1Enabled) {%>
|
||||
org.apache.log4j.Logger.getLogger("audit").setLevel(org.apache.log4j.Level.DEBUG);
|
||||
<%}%>
|
||||
|
||||
<%if(isLog4j2Enabled) {%>
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel("audit", org.apache.logging.log4j.Level.DEBUG);
|
||||
<%}%>
|
||||
auditLogger_<%=jobCatcherNode.getUniqueName()%> = org.talend.job.audit.JobEventAuditLoggerFactory.createJobAuditLogger(properties_<%=jobCatcherNode.getUniqueName()%>);
|
||||
}
|
||||
<%
|
||||
@@ -768,6 +803,25 @@ if(execStat) {
|
||||
|
||||
startTime = System.currentTimeMillis();
|
||||
<%
|
||||
|
||||
boolean isCustomProxySettingsUsed = false;
|
||||
List<? extends INode> proxyNodes = process.getNodesOfType("tSetProxy");
|
||||
for (int i = 0; i < proxyNodes.size(); i++) {
|
||||
INode proxyNode = proxyNodes.get(i);
|
||||
String useCustomProxyCheckboxValue = ElementParameterParser.getValue(proxyNode, "__PROXY_GLOBAL__");
|
||||
|
||||
if (useCustomProxyCheckboxValue != null && !"true".equals(useCustomProxyCheckboxValue)) {
|
||||
isCustomProxySettingsUsed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (isCustomProxySettingsUsed) {
|
||||
%>
|
||||
java.net.ProxySelector proxySelectorOnStartJob = java.net.ProxySelector.getDefault();
|
||||
<%
|
||||
}
|
||||
|
||||
//1. send the begin msg
|
||||
|
||||
for (INode statCatcherNode : process.getNodesOfType("tStatCatcher")) {
|
||||
@@ -990,6 +1044,13 @@ this.globalResumeTicket = true;//to run tPostJob
|
||||
|
||||
<%= ElementParameterParser.getValue(process, "__FOOTER_CODE__")%>
|
||||
|
||||
<%
|
||||
if (isCustomProxySettingsUsed) {
|
||||
%>
|
||||
java.net.ProxySelector.setDefault(proxySelectorOnStartJob);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
end = System.currentTimeMillis();
|
||||
|
||||
if (watch) {
|
||||
@@ -1509,12 +1570,10 @@ if (execStat) {
|
||||
}
|
||||
} else if (arg.startsWith("--log4jLevel=")) {
|
||||
log4jLevel = arg.substring(13);
|
||||
} else if (arg.startsWith("--monitoring=")) {//for trunjob call
|
||||
enableLogStash = "true".equalsIgnoreCase(arg.substring(13));
|
||||
}
|
||||
|
||||
if(!enableLogStash) {
|
||||
enableLogStash = "true".equalsIgnoreCase(System.getProperty("monitoring"));
|
||||
} else if (arg.startsWith("--monitoring") && arg.contains("=")) {//for trunjob call
|
||||
final int equal = arg.indexOf('=');
|
||||
final String key = arg.substring("--".length(), equal);
|
||||
System.setProperty(key, arg.substring(equal + 1));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -139,9 +139,11 @@ for (INode node : process.getNodesOfType("tRESTClient")) {
|
||||
}
|
||||
|
||||
boolean talendEsbJobFactory = actAsProvider || !process.getNodesOfType("tRouteInput").isEmpty();
|
||||
boolean talendEsbJob = talendEsbJobFactory || actAsConsumer || ProcessorUtilities.isEsbJob(process.getId(), process.getVersion());
|
||||
boolean talendEsbJob = talendEsbJobFactory || actAsConsumer || ProcessorUtilities.isEsbJob(process);
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(process, "__LOG4J_ACTIVATE__"));
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(process, "__LOG4J_ACTIVATE__"));//log4j enable
|
||||
boolean isLog4j2Enabled = ("true").equals(ElementParameterParser.getValue(process, "__LOG4J2_ACTIVATE__"));//log4j2 enable
|
||||
boolean isLog4j1Enabled = !isLog4j2Enabled;//log4j1 enable
|
||||
|
||||
if (talendMdmJob) {
|
||||
talendJobInterfaces += ", TalendMDMJob"; // Talend MDM job
|
||||
@@ -156,7 +158,14 @@ if (talendEsbJobFactory) {
|
||||
public class <%=className%> implements <%=talendJobInterfaces%> {
|
||||
<%if(isLog4jEnabled){%>
|
||||
static {System.setProperty("TalendJob.log", "<%=className%>.log");}
|
||||
|
||||
<%if(isLog4j1Enabled){%>
|
||||
private static org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(<%=className%>.class);
|
||||
<%}%>
|
||||
|
||||
<%if(isLog4j2Enabled){%>
|
||||
private static org.apache.logging.log4j.Logger log = org.apache.logging.log4j.LogManager.getLogger(<%=className%>.class);
|
||||
<%}%>
|
||||
<%}%>
|
||||
|
||||
protected static void logIgnoredError(String message, Throwable cause) {
|
||||
|
||||
@@ -94,7 +94,7 @@ public class JavaRoutineSynchronizer extends AbstractRoutineSynchronizer {
|
||||
|
||||
private void syncRoutineItems(Collection<RoutineItem> routineObjects, boolean forceUpdate) throws SystemException {
|
||||
for (RoutineItem routineItem : routineObjects) {
|
||||
syncRoutine(routineItem, true, true, forceUpdate);
|
||||
syncRoutine(routineItem, true, forceUpdate);
|
||||
}
|
||||
syncSystemRoutine(ProjectManager.getInstance().getCurrentProject());
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<groupId>com.microsoft.azure</groupId>
|
||||
<artifactId>adal4j</artifactId>
|
||||
|
||||
<version>1.1.1-patch</version>
|
||||
<version>1.1.1-20191012</version>
|
||||
<packaging>jar</packaging>
|
||||
<name>adal4j</name>
|
||||
<description>
|
||||
@@ -105,7 +105,7 @@
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.5</version>
|
||||
<version>1.7.25</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
|
||||
@@ -33,10 +33,15 @@
|
||||
<version>3.0.1</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.25</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<version>1.7.5</version>
|
||||
<version>1.7.25</version>
|
||||
</dependency>
|
||||
<!-- Spring 3 dependencies -->
|
||||
<dependency>
|
||||
|
||||
@@ -27,7 +27,8 @@ import java.util.List;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import org.apache.commons.lang3.text.WordUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
@@ -41,7 +42,7 @@ import org.json.JSONObject;
|
||||
*/
|
||||
public class JSONHelper {
|
||||
|
||||
private static Logger logger = Logger.getLogger(JSONHelper.class);
|
||||
private static Logger logger = LoggerFactory.getLogger(JSONHelper.class);
|
||||
|
||||
JSONHelper() {
|
||||
// PropertyConfigurator.configure("log4j.properties");
|
||||
|
||||
@@ -1,69 +1,69 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>checkArchive-1.1-20181130</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<name>checkArchive</name>
|
||||
<description>Dependence for tFileArchive and tFileUnAchive</description>
|
||||
<url>http://maven.apache.org</url>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<java.source.version>1.7</java.source.version>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
</snapshotRepository>
|
||||
<repository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-compress -->
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<version>1.10</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/java</directory>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>${java.source.version}</source>
|
||||
<target>${java.source.version}</target>
|
||||
<showDeprecation>true</showDeprecation>
|
||||
<showWarnings>true</showWarnings>
|
||||
<compilerArgument>-XDignore.symbol.file</compilerArgument>
|
||||
<fork>true</fork>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>checkArchive-1.1-20190917</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<name>checkArchive</name>
|
||||
<description>Dependence for tFileArchive and tFileUnAchive</description>
|
||||
<url>http://maven.apache.org</url>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<java.source.version>1.7</java.source.version>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
</snapshotRepository>
|
||||
<repository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-compress -->
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<version>1.19</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/java</directory>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>${java.source.version}</source>
|
||||
<target>${java.source.version}</target>
|
||||
<showDeprecation>true</showDeprecation>
|
||||
<showWarnings>true</showWarnings>
|
||||
<compilerArgument>-XDignore.symbol.file</compilerArgument>
|
||||
<fork>true</fork>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>commons-net-ftps-proxy</artifactId>
|
||||
<version>3.6.1-talend-20190128</version>
|
||||
<version>3.6.1-talend-20190819</version>
|
||||
|
||||
<name>commons-net-talend</name>
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package org.talend.ftp;
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Method;
|
||||
import java.net.Socket;
|
||||
import java.net.InetAddress;
|
||||
import java.util.Locale;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
@@ -29,11 +30,13 @@ public class SSLSessionReuseFTPSClient extends FTPSClient {
|
||||
final Object cache = sessionHostPortCache.get(context);
|
||||
final Method putMethod = cache.getClass().getDeclaredMethod("put", Object.class, Object.class);
|
||||
putMethod.setAccessible(true);
|
||||
final Method getHostMethod = socket.getClass().getDeclaredMethod("getHost");
|
||||
getHostMethod.setAccessible(true);
|
||||
Object host = getHostMethod.invoke(socket);
|
||||
final String key =
|
||||
String.format("%s:%s", host, String.valueOf(socket.getPort())).toLowerCase(Locale.ROOT);
|
||||
InetAddress address = socket.getInetAddress();
|
||||
int port = socket.getPort();
|
||||
|
||||
String key = String.format("%s:%s", address.getHostName(), String.valueOf(port)).toLowerCase(Locale.ROOT);
|
||||
putMethod.invoke(cache, key, session);
|
||||
|
||||
key = String.format("%s:%s", address.getHostAddress(), String.valueOf(port)).toLowerCase(Locale.ROOT);
|
||||
putMethod.invoke(cache, key, session);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<groupId>net.sf.json-lib</groupId>
|
||||
<artifactId>json-lib</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<version>2.4.1-talend</version>
|
||||
<version>2.4.4-talend</version>
|
||||
<name>json-lib</name>
|
||||
|
||||
<properties>
|
||||
|
||||
@@ -25,12 +25,12 @@ import java.util.Iterator;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import net.sf.json.util.JSONUtils;
|
||||
import net.sf.json.util.JsonEventListener;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import net.sf.json.util.JSONUtils;
|
||||
import net.sf.json.util.JsonEventListener;
|
||||
|
||||
/**
|
||||
* Base class for JSONObject and JSONArray.
|
||||
*
|
||||
@@ -227,7 +227,8 @@ abstract class AbstractJSON implements JSON {
|
||||
}
|
||||
|
||||
protected Object _processValue( Object value, JsonConfig jsonConfig ) {
|
||||
if( JSONNull.getInstance().equals( value ) ) {
|
||||
if( JSONNull.getInstance().equals( value ) &&
|
||||
((jsonConfig.getJsonStandard() != JsonStandard.WRAP_NULL_STRINGS) || !"null".equals(value))) {
|
||||
return JSONNull.getInstance();
|
||||
} else if( Class.class.isAssignableFrom( value.getClass() ) || value instanceof Class ) {
|
||||
return ((Class) value).getName();
|
||||
|
||||
@@ -2391,7 +2391,7 @@ public final class JSONObject extends AbstractJSON implements JSON, Map, Compara
|
||||
}
|
||||
try{
|
||||
Iterator keys = keys();
|
||||
StringBuffer sb = new StringBuffer( "{" );
|
||||
StringBuilder sb = new StringBuilder( "{" );
|
||||
|
||||
while( keys.hasNext() ){
|
||||
if( sb.length() > 1 ){
|
||||
@@ -2460,7 +2460,7 @@ public final class JSONObject extends AbstractJSON implements JSON, Map, Compara
|
||||
return this.toString();
|
||||
}
|
||||
Iterator keys = keys();
|
||||
StringBuffer sb = new StringBuffer( "{" );
|
||||
StringBuilder sb = new StringBuilder( "{" );
|
||||
int newindent = indent + indentFactor;
|
||||
Object o;
|
||||
if( n == 1 ){
|
||||
@@ -2549,7 +2549,7 @@ public final class JSONObject extends AbstractJSON implements JSON, Map, Compara
|
||||
if( o instanceof JSONArray ){
|
||||
((JSONArray) o).element( value, jsonConfig );
|
||||
}else{
|
||||
setInternal( key, new JSONArray().element( o )
|
||||
setInternal( key, new JSONArray().element( o, jsonConfig )
|
||||
.element( value, jsonConfig ), jsonConfig );
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,7 +86,6 @@ public class JsonConfig {
|
||||
private boolean handleJettisonEmptyElement;
|
||||
private boolean handleJettisonSingleElementArray;
|
||||
private boolean ignoreDefaultExcludes;
|
||||
//private boolean ignoreJPATransient;
|
||||
private boolean ignoreTransientFields;
|
||||
private boolean ignorePublicFields = true;
|
||||
private boolean javascriptCompliant;
|
||||
@@ -110,6 +109,7 @@ public class JsonConfig {
|
||||
private Map typeMap = new HashMap();
|
||||
private List ignoreFieldAnnotations = new ArrayList();
|
||||
private boolean allowNonStringKeys = false;
|
||||
private JsonStandard jsonStandard = JsonStandard.LEGACY;
|
||||
|
||||
public JsonConfig() {
|
||||
}
|
||||
@@ -1246,7 +1246,20 @@ public class JsonConfig {
|
||||
this.newBeanInstanceStrategy = newBeanInstanceStrategy == null ? DEFAULT_NEW_BEAN_INSTANCE_STRATEGY
|
||||
: newBeanInstanceStrategy;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sets the config to wrap "null" strings as strings instead of JsonNull.
|
||||
*
|
||||
*/
|
||||
|
||||
public void setJsonStandard(JsonStandard wrapNullStringValues) {
|
||||
this.jsonStandard = wrapNullStringValues;
|
||||
}
|
||||
|
||||
public JsonStandard getJsonStandard() {
|
||||
return jsonStandard;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets a PropertyExclusionClassMatcher to use.<br>
|
||||
* Will set default value (PropertyExclusionClassMatcher.DEFAULT) if null.<br>
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package net.sf.json;
|
||||
|
||||
public enum JsonStandard {
|
||||
/**
|
||||
* Out of date standard used to be default before
|
||||
*/
|
||||
LEGACY,
|
||||
|
||||
/**
|
||||
* Updated standard due to RFC 7159 to not unwrap "null" strings (keep quotations)
|
||||
*/
|
||||
WRAP_NULL_STRINGS
|
||||
|
||||
}
|
||||
@@ -15,6 +15,7 @@
|
||||
*/
|
||||
package net.sf.json.util;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import net.sf.json.JSONArray;
|
||||
import net.sf.json.JSONException;
|
||||
import net.sf.json.JSONNull;
|
||||
@@ -414,7 +415,7 @@ public class JSONTokener {
|
||||
}
|
||||
|
||||
try{
|
||||
return NumberUtils.createNumber(s);
|
||||
return createNumber(s);
|
||||
}catch( Exception e ){
|
||||
return s;
|
||||
}
|
||||
@@ -435,6 +436,26 @@ public class JSONTokener {
|
||||
return s;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method has been added to fix https://jira.talendforge.org/browse/TDI-42689
|
||||
*
|
||||
* @param s The String representation of the number
|
||||
* @return The Number instance
|
||||
*/
|
||||
private Number createNumber(String s){
|
||||
boolean isDecimal = s.indexOf('.') != -1;
|
||||
|
||||
if(isDecimal){
|
||||
Double d = Double.valueOf(s);
|
||||
if(Double.POSITIVE_INFINITY == Math.abs(d)){
|
||||
return new BigDecimal(s);
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
return NumberUtils.createNumber(s);
|
||||
}
|
||||
|
||||
/**
|
||||
* Look at the next character in the source string.
|
||||
*
|
||||
|
||||
@@ -24,6 +24,8 @@ import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.beanutils.DynaBean;
|
||||
|
||||
import net.sf.ezmorph.MorphUtils;
|
||||
import net.sf.ezmorph.MorpherRegistry;
|
||||
import net.sf.ezmorph.bean.MorphDynaBean;
|
||||
@@ -36,10 +38,9 @@ import net.sf.json.JSONNull;
|
||||
import net.sf.json.JSONObject;
|
||||
import net.sf.json.JSONString;
|
||||
import net.sf.json.JsonConfig;
|
||||
import net.sf.json.JsonStandard;
|
||||
import net.sf.json.regexp.RegexpUtils;
|
||||
|
||||
import org.apache.commons.beanutils.DynaBean;
|
||||
|
||||
/**
|
||||
* Provides useful methods on java objects and JSON values.
|
||||
*
|
||||
@@ -124,7 +125,7 @@ public final class JSONUtils {
|
||||
public static String getFunctionBody( String function ) {
|
||||
return RegexpUtils.getMatcher( FUNCTION_BODY_PATTERN, true ).getGroupIfMatches( function, 1 );
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the params of a function literal.
|
||||
*/
|
||||
@@ -184,14 +185,12 @@ public final class JSONUtils {
|
||||
return Integer.class;
|
||||
}else if( isLong( n ) ){
|
||||
return Long.class;
|
||||
}else if( isFloat( n ) ){
|
||||
return Float.class;
|
||||
}else if( isBigInteger( n ) ){
|
||||
return BigInteger.class;
|
||||
}else if( isBigDecimal( n ) ){
|
||||
return BigDecimal.class;
|
||||
}else if( isDouble( n ) ){
|
||||
return Double.class;
|
||||
}else if( isBigDecimal( n ) ){
|
||||
return BigDecimal.class;
|
||||
}else{
|
||||
throw new JSONException( "Unsupported type" );
|
||||
}
|
||||
@@ -607,7 +606,7 @@ public final class JSONUtils {
|
||||
return input.startsWith( SINGLE_QUOTE ) && input.endsWith( SINGLE_QUOTE ) ||
|
||||
input.startsWith( DOUBLE_QUOTE ) && input.endsWith( DOUBLE_QUOTE );
|
||||
}
|
||||
|
||||
|
||||
public static boolean isJsonKeyword( String input, JsonConfig jsonConfig ) {
|
||||
if( input == null ){
|
||||
return false;
|
||||
@@ -617,7 +616,7 @@ public final class JSONUtils {
|
||||
"false".equals( input ) ||
|
||||
(jsonConfig.isJavascriptCompliant() && "undefined".equals( input ));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Throw an exception if the object is an NaN or infinite number.
|
||||
*
|
||||
@@ -756,7 +755,77 @@ public final class JSONUtils {
|
||||
return quote( value.toString() );
|
||||
}
|
||||
|
||||
/**
|
||||
public static String jsonToStandardizedString(JSON json, JsonStandard standard) {
|
||||
switch (standard) {
|
||||
case WRAP_NULL_STRINGS:
|
||||
if (json.isArray()) {
|
||||
JSONArray jsonArray = (JSONArray) json;
|
||||
return jsonArrayToWrappedNullStrings(jsonArray);
|
||||
} else if (!JSONNull.getInstance().equals(json)) {
|
||||
return jsonToWrappedNullStrings((JSONObject) json);
|
||||
}
|
||||
default:
|
||||
return json.toString();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @return plain String from JSONObject (@see JSONObject#toString()), but wrap null strings to quotation
|
||||
*/
|
||||
private static String jsonToWrappedNullStrings(JSONObject json) {
|
||||
if (json.isNullObject()) {
|
||||
return JSONNull.getInstance()
|
||||
.toString();
|
||||
}
|
||||
try {
|
||||
Iterator keys = json.keys();
|
||||
StringBuilder sb = new StringBuilder("{");
|
||||
|
||||
while (keys.hasNext()) {
|
||||
if (sb.length() > 1) {
|
||||
sb.append(',');
|
||||
}
|
||||
Object o = keys.next();
|
||||
sb.append(quote(o.toString()));
|
||||
sb.append(':');
|
||||
sb.append(valueToStringWrappedNullStrings(json.get(o)));
|
||||
}
|
||||
sb.append('}');
|
||||
return sb.toString();
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static String jsonArrayToWrappedNullStrings(JSONArray jsonArray) {
|
||||
final String separator = ",";
|
||||
StringBuilder sb = new StringBuilder("[");
|
||||
|
||||
for (int i = 0; i < jsonArray.size(); i++) {
|
||||
if (i > 0) {
|
||||
sb.append(separator);
|
||||
}
|
||||
sb.append(JSONUtils.valueToStringWrappedNullStrings(jsonArray.get(i)));
|
||||
}
|
||||
|
||||
return sb.append("]").toString();
|
||||
}
|
||||
|
||||
private static String valueToStringWrappedNullStrings(Object o) {
|
||||
if ("null".equals(o)) {
|
||||
return quote(o.toString());
|
||||
} else if (o instanceof JSONArray) {
|
||||
return jsonArrayToWrappedNullStrings((JSONArray) o);
|
||||
} else if (o instanceof JSONObject) {
|
||||
return jsonToWrappedNullStrings((JSONObject) o);
|
||||
} else {
|
||||
return valueToString(o);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds out if n represents a BigInteger
|
||||
*
|
||||
* @return true if n is instanceOf BigInteger or the literal value can be
|
||||
|
||||
@@ -22,6 +22,8 @@ import net.sf.json.JSONException;
|
||||
import net.sf.json.JSONFunction;
|
||||
import net.sf.json.JSONNull;
|
||||
import net.sf.json.JSONObject;
|
||||
import net.sf.json.JsonConfig;
|
||||
import net.sf.json.JsonStandard;
|
||||
import net.sf.json.util.JSONUtils;
|
||||
import nu.xom.Attribute;
|
||||
import nu.xom.Builder;
|
||||
@@ -29,7 +31,6 @@ import nu.xom.Document;
|
||||
import nu.xom.Element;
|
||||
import nu.xom.Elements;
|
||||
import nu.xom.Node;
|
||||
import nu.xom.ProcessingInstruction;
|
||||
import nu.xom.Serializer;
|
||||
import nu.xom.Text;
|
||||
import org.apache.commons.lang.ArrayUtils;
|
||||
@@ -163,6 +164,15 @@ public class XMLSerializer {
|
||||
*/
|
||||
private boolean useLongDecimals;
|
||||
|
||||
/**
|
||||
* The config parameter to wrap "null" strings as strings instead of JsonNull.
|
||||
*/
|
||||
private JsonStandard jsonStandard;
|
||||
/**
|
||||
* flag for if parse empty elements as empty strings
|
||||
*/
|
||||
private boolean useEmptyStrings;
|
||||
|
||||
/**
|
||||
* Creates a new XMLSerializer with default options.<br>
|
||||
* <ul>
|
||||
@@ -367,17 +377,19 @@ public class XMLSerializer {
|
||||
return JSONNull.getInstance();
|
||||
}
|
||||
String defaultType = getType( root, JSONTypes.STRING );
|
||||
JsonConfig config = new JsonConfig();
|
||||
config.setJsonStandard(jsonStandard);
|
||||
if( isArray( root, true ) ){
|
||||
json = processArrayElement( root, defaultType );
|
||||
if( forceTopLevelObject ){
|
||||
String key = removeNamespacePrefix( root.getQualifiedName() );
|
||||
json = new JSONObject().element( key, json );
|
||||
json = new JSONObject().element(key, json, config);
|
||||
}
|
||||
}else{
|
||||
json = processObjectElement( root, defaultType );
|
||||
if( forceTopLevelObject ){
|
||||
String key = removeNamespacePrefix( root.getQualifiedName() );
|
||||
json = new JSONObject().element( key, json );
|
||||
json = new JSONObject().element(key, json, config);
|
||||
}
|
||||
}
|
||||
}catch( JSONException jsone ){
|
||||
@@ -655,6 +667,10 @@ public class XMLSerializer {
|
||||
this.useLongDecimals = useLongDecimals;
|
||||
}
|
||||
|
||||
public void setJsonStandard(JsonStandard jsonStandard) {
|
||||
this.jsonStandard = jsonStandard;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a JSON value into a XML string with UTF-8 encoding.<br>
|
||||
*
|
||||
@@ -803,6 +819,8 @@ public class XMLSerializer {
|
||||
clazz = JSONTypes.OBJECT;
|
||||
}else if( JSONTypes.ARRAY.compareToIgnoreCase( clazzText ) == 0 ){
|
||||
clazz = JSONTypes.ARRAY;
|
||||
} else if(JSONTypes.STRING.equalsIgnoreCase(clazzText)) {
|
||||
clazz = JSONTypes.STRING;
|
||||
}
|
||||
}
|
||||
return clazz;
|
||||
@@ -1257,14 +1275,16 @@ public class XMLSerializer {
|
||||
}
|
||||
|
||||
private void setOrAccumulate( JSONObject jsonObject, String key, Object value ) {
|
||||
JsonConfig config = new JsonConfig();
|
||||
config.setJsonStandard(jsonStandard);
|
||||
if( jsonObject.has( key ) ){
|
||||
jsonObject.accumulate( key, value );
|
||||
jsonObject.accumulate(key, value, config);
|
||||
Object val = jsonObject.get( key );
|
||||
if( val instanceof JSONArray ){
|
||||
((JSONArray) val).setExpandElements( true );
|
||||
}
|
||||
}else{
|
||||
jsonObject.element( key, value );
|
||||
jsonObject.element( key, value, config);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1272,9 +1292,10 @@ public class XMLSerializer {
|
||||
String clazz = getClass( element );
|
||||
String type = getType( element );
|
||||
type = (type == null) ? defaultType : type;
|
||||
|
||||
JsonConfig config = new JsonConfig();
|
||||
config.setJsonStandard(jsonStandard);
|
||||
if( hasNamespaces( element ) && !skipNamespaces ){
|
||||
jsonArray.element( simplifyValue( null, processElement( element, type ) ) );
|
||||
jsonArray.element(simplifyValue(null, processElement(element, type)), config);
|
||||
return;
|
||||
}else if( element.getAttributeCount() > 0 ){
|
||||
if( isFunction( element ) ){
|
||||
@@ -1282,10 +1303,10 @@ public class XMLSerializer {
|
||||
String[] params = null;
|
||||
String text = element.getValue();
|
||||
params = StringUtils.split( paramsAttribute.getValue(), "," );
|
||||
jsonArray.element( new JSONFunction( params, text ) );
|
||||
jsonArray.element(new JSONFunction(params, text));
|
||||
return;
|
||||
}else{
|
||||
jsonArray.element( simplifyValue( null, processElement( element, type ) ) );
|
||||
jsonArray.element(simplifyValue(null, processElement(element, type)), config);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -1293,10 +1314,10 @@ public class XMLSerializer {
|
||||
boolean classProcessed = false;
|
||||
if( clazz != null ){
|
||||
if( clazz.compareToIgnoreCase( JSONTypes.ARRAY ) == 0 ){
|
||||
jsonArray.element( processArrayElement( element, type ) );
|
||||
jsonArray.element(processArrayElement(element, type), config);
|
||||
classProcessed = true;
|
||||
}else if( clazz.compareToIgnoreCase( JSONTypes.OBJECT ) == 0 ){
|
||||
jsonArray.element( simplifyValue( null, processObjectElement( element, type ) ) );
|
||||
jsonArray.element(simplifyValue(null, processObjectElement( element, type)), config);
|
||||
classProcessed = true;
|
||||
}
|
||||
}
|
||||
@@ -1332,12 +1353,12 @@ public class XMLSerializer {
|
||||
jsonArray.element( new JSONFunction( params, text ) );
|
||||
}else{
|
||||
if( isArray( element, false ) ){
|
||||
jsonArray.element( processArrayElement( element, defaultType ) );
|
||||
jsonArray.element(processArrayElement(element, defaultType), config);
|
||||
}else if( isObject( element, false ) ){
|
||||
jsonArray.element( simplifyValue( null, processObjectElement( element,
|
||||
defaultType ) ) );
|
||||
jsonArray.element(simplifyValue(null, processObjectElement(element,
|
||||
defaultType)), config);
|
||||
}else{
|
||||
jsonArray.element( trimSpaceFromValue( element.getValue() ) );
|
||||
jsonArray.element(trimSpaceFromValue(element.getValue()), config);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1414,6 +1435,8 @@ public class XMLSerializer {
|
||||
String text = element.getValue();
|
||||
params = StringUtils.split( paramsAttribute.getValue(), "," );
|
||||
setOrAccumulate( jsonObject, key, new JSONFunction( params, text ) );
|
||||
} else if( useEmptyStrings && clazz != null && clazz.equalsIgnoreCase(JSONTypes.STRING) ) {
|
||||
setTextValue(jsonObject, key, element);
|
||||
}else{
|
||||
if( isArray( element, false ) ){
|
||||
setOrAccumulate( jsonObject, key, processArrayElement( element, defaultType ) );
|
||||
@@ -1421,19 +1444,23 @@ public class XMLSerializer {
|
||||
setOrAccumulate( jsonObject, key, simplifyValue( jsonObject,
|
||||
processObjectElement( element, defaultType ) ) );
|
||||
}else{
|
||||
String value;
|
||||
if( isKeepCData && isCData( element ) ){
|
||||
value = "<![CDATA[" + element.getValue() + "]]>";
|
||||
}else{
|
||||
value = element.getValue();
|
||||
}
|
||||
setOrAccumulate( jsonObject, key, trimSpaceFromValue( value ) );
|
||||
setTextValue(jsonObject, key, element);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void setTextValue(final JSONObject jsonObject, final String key, final Element element) {
|
||||
String value;
|
||||
if( isKeepCData && isCData( element ) ){
|
||||
value = "<![CDATA[" + element.getValue() + "]]>";
|
||||
}else{
|
||||
value = element.getValue();
|
||||
}
|
||||
setOrAccumulate( jsonObject, key, trimSpaceFromValue( value ) );
|
||||
}
|
||||
|
||||
private boolean isCData( Element element ) {
|
||||
if( element.getChildCount() == 1 ){
|
||||
final Node child = element.getChild( 0 );
|
||||
@@ -1493,6 +1520,14 @@ public class XMLSerializer {
|
||||
return str;
|
||||
}
|
||||
|
||||
public void setUseEmptyStrings(boolean useEmptyStrings) {
|
||||
this.useEmptyStrings = useEmptyStrings;
|
||||
}
|
||||
|
||||
public boolean isUseEmptyStrings() {
|
||||
return this.useEmptyStrings;
|
||||
}
|
||||
|
||||
private static class CustomElement extends Element {
|
||||
private static String getName( String name ) {
|
||||
int colon = name.indexOf( ':' );
|
||||
|
||||
@@ -27,6 +27,7 @@ import net.sf.json.processors.PropertyNameProcessor;
|
||||
import net.sf.json.sample.BeanA;
|
||||
import net.sf.json.sample.BeanB;
|
||||
import net.sf.json.sample.BeanC;
|
||||
import net.sf.json.sample.BeanD;
|
||||
import net.sf.json.sample.BeanFoo;
|
||||
import net.sf.json.sample.BeanWithFunc;
|
||||
import net.sf.json.sample.ChildBean;
|
||||
@@ -1042,6 +1043,17 @@ public class TestJSONObject extends TestCase {
|
||||
JSONArray.toArray( jsonObject.getJSONArray( "intarray" ) ) );
|
||||
}
|
||||
|
||||
public void testToBean_BeanD() {
|
||||
String json = "{bool:true,integer:1,string:\"json\",doublearray:[4.2424245783E7, 123456789.2424245783E7, 6.0]}";
|
||||
JSONObject jsonObject = JSONObject.fromObject( json );
|
||||
BeanD bean = (BeanD) JSONObject.toBean( jsonObject, BeanD.class );
|
||||
assertEquals( jsonObject.get( "bool" ), Boolean.valueOf( bean.isBool() ) );
|
||||
assertEquals( jsonObject.get( "integer" ), new Integer( bean.getInteger() ) );
|
||||
assertEquals( jsonObject.get( "string" ), bean.getString() );
|
||||
Assertions.assertEquals( bean.getDoublearray(),
|
||||
JSONArray.toArray( jsonObject.getJSONArray( "doublearray" ) ) );
|
||||
}
|
||||
|
||||
public void testToBean_ClassBean() {
|
||||
JSONObject json = new JSONObject();
|
||||
json.element( "klass", "java.lang.Object" );
|
||||
@@ -1050,9 +1062,29 @@ public class TestJSONObject extends TestCase {
|
||||
assertEquals( Object.class, bean.getKlass() );
|
||||
}
|
||||
|
||||
public void testToBean_DynaBean__BigInteger_BigDecimal() {
|
||||
public void testToBean_DynaBean__BigInteger_Double() {
|
||||
BigInteger l = new BigDecimal( "1.7976931348623157E308" ).toBigInteger();
|
||||
BigDecimal m = new BigDecimal( "1.7976931348623157E307" ).add( new BigDecimal( "0.0001" ) );
|
||||
JSONObject json = new JSONObject().element( "i", BigInteger.ZERO )
|
||||
.element( "d", MorphUtils.BIGDECIMAL_ONE )
|
||||
.element( "bi", l )
|
||||
.element( "bd", m );
|
||||
Object bean = JSONObject.toBean( json );
|
||||
Object i = ((MorphDynaBean) bean).get( "i" );
|
||||
Object d = ((MorphDynaBean) bean).get( "d" );
|
||||
assertTrue( i instanceof Integer );
|
||||
assertTrue( d instanceof Integer );
|
||||
|
||||
Object bi = ((MorphDynaBean) bean).get( "bi" );
|
||||
Object bd = ((MorphDynaBean) bean).get( "bd" );
|
||||
assertTrue( bi instanceof BigInteger );
|
||||
assertTrue( bd instanceof Double );
|
||||
}
|
||||
|
||||
|
||||
public void testToBean_DynaBean__BigInteger_BigDecimal() {
|
||||
BigInteger l = new BigDecimal( "1.7976931348623157E308" ).toBigInteger();
|
||||
BigDecimal m = new BigDecimal( "-1.7976931348623157E309" ).add( new BigDecimal( "0.0001" ) );
|
||||
JSONObject json = new JSONObject().element( "i", BigInteger.ZERO )
|
||||
.element( "d", MorphUtils.BIGDECIMAL_ONE )
|
||||
.element( "bi", l )
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright 2002-2009 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package net.sf.json.sample;
|
||||
|
||||
import org.apache.commons.lang.builder.ToStringBuilder;
|
||||
import org.apache.commons.lang.builder.ToStringStyle;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
|
||||
/**
|
||||
* @author Andres Almiray <aalmiray@users.sourceforge.net>
|
||||
*/
|
||||
public class BeanD extends BeanA {
|
||||
private Double[] doublearray = new Double[3];
|
||||
|
||||
public Double[] getDoublearray() {
|
||||
return doublearray;
|
||||
}
|
||||
|
||||
public void setDoublearray(Double[] doublearray) {
|
||||
this.doublearray = doublearray;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return ToStringBuilder.reflectionToString( this, ToStringStyle.MULTI_LINE_STYLE );
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@
|
||||
package net.sf.json.util;
|
||||
|
||||
import java.io.StringWriter;
|
||||
import java.math.BigDecimal;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
import net.sf.json.JSONFunction;
|
||||
@@ -93,7 +94,7 @@ public class TestJSONBuilder extends TestCase {
|
||||
.endObject();
|
||||
JSONObject jsonObj = JSONObject.fromObject( w.toString() );
|
||||
assertEquals( Boolean.TRUE, jsonObj.get( "bool" ) );
|
||||
assertEquals( new Double( 1.1d ), jsonObj.get( "numDouble" ) );
|
||||
assertEquals( Double.valueOf( "1.1" ), jsonObj.get( "numDouble" ) );
|
||||
assertEquals( new Long( 2 ).longValue(), ((Number) jsonObj.get( "numInt" )).longValue() );
|
||||
assertEquals( "text", jsonObj.get( "text" ) );
|
||||
assertTrue( JSONUtils.isFunction( jsonObj.get( "func" ) ) );
|
||||
|
||||
@@ -20,6 +20,8 @@ import junit.framework.TestCase;
|
||||
import net.sf.json.JSONFunction;
|
||||
import net.sf.json.JSONObject;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
|
||||
/**
|
||||
* @author Andres Almiray <aalmiray@users.sourceforge.net>
|
||||
*/
|
||||
@@ -85,7 +87,7 @@ public class TestJSONStringer extends TestCase {
|
||||
.endObject();
|
||||
JSONObject jsonObj = JSONObject.fromObject( b.toString() );
|
||||
assertEquals( Boolean.TRUE, jsonObj.get( "bool" ) );
|
||||
assertEquals( new Double( 1.1d ), jsonObj.get( "numDouble" ) );
|
||||
assertEquals( Double.valueOf( "1.1" ), jsonObj.get( "numDouble" ) );
|
||||
assertEquals( new Long( 2 ).longValue(), ((Number) jsonObj.get( "numInt" )).longValue() );
|
||||
assertEquals( "text", jsonObj.get( "text" ) );
|
||||
assertTrue( JSONUtils.isFunction( jsonObj.get( "func" ) ) );
|
||||
|
||||
@@ -17,16 +17,25 @@
|
||||
package net.sf.json.util;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
import net.sf.json.JSONArray;
|
||||
import net.sf.json.JSONException;
|
||||
import net.sf.json.JSONFunction;
|
||||
import net.sf.json.JSONObject;
|
||||
import net.sf.json.JsonConfig;
|
||||
import net.sf.json.JsonStandard;
|
||||
|
||||
/**
|
||||
* @author Andres Almiray <aalmiray@users.sourceforge.net>
|
||||
*/
|
||||
public class TestJSONUtils extends TestCase {
|
||||
private static Map<String, String> valuesMap;
|
||||
|
||||
public static void main( String[] args ) {
|
||||
junit.textui.TestRunner.run( TestJSONUtils.class );
|
||||
}
|
||||
@@ -35,6 +44,14 @@ public class TestJSONUtils extends TestCase {
|
||||
super( name );
|
||||
}
|
||||
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
valuesMap = new LinkedHashMap<>();
|
||||
|
||||
valuesMap.put("key1", "null");
|
||||
valuesMap.put("key2", "not_null");
|
||||
}
|
||||
|
||||
public void testDoubleToString_infinite() {
|
||||
assertEquals( "null", JSONUtils.doubleToString( Double.POSITIVE_INFINITY ) );
|
||||
}
|
||||
@@ -177,4 +194,48 @@ public class TestJSONUtils extends TestCase {
|
||||
// ok
|
||||
}
|
||||
}
|
||||
|
||||
public void testNullStringsWrapped() {
|
||||
JsonConfig config = new JsonConfig();
|
||||
config.setJsonStandard(JsonStandard.WRAP_NULL_STRINGS);
|
||||
|
||||
JSONObject jsonObject = new JSONObject();
|
||||
jsonObject.putAll(valuesMap, config);
|
||||
|
||||
String resultingString = JSONUtils.jsonToStandardizedString(jsonObject, JsonStandard.WRAP_NULL_STRINGS);
|
||||
assertFalse("Wrapping null strings standard's broken", Objects.equals(jsonObject.toString(), resultingString));
|
||||
assertTrue(resultingString.contains("\"null\""));
|
||||
}
|
||||
|
||||
public void testNullStringsUnwrapped() {
|
||||
JSONObject jsonObject = new JSONObject();
|
||||
jsonObject.putAll(valuesMap);
|
||||
|
||||
String resultingString = JSONUtils.jsonToStandardizedString(jsonObject, JsonStandard.WRAP_NULL_STRINGS);
|
||||
assertEquals(jsonObject.toString(), resultingString);
|
||||
}
|
||||
|
||||
public void testNullStringsOnArrayWrapped() {
|
||||
JsonConfig config = new JsonConfig();
|
||||
config.setJsonStandard(JsonStandard.WRAP_NULL_STRINGS);
|
||||
|
||||
JSONArray jsonArray = new JSONArray();
|
||||
jsonArray.add("abc");
|
||||
jsonArray.add("null", config);
|
||||
jsonArray.add(null);
|
||||
|
||||
String resultingString = JSONUtils.jsonToStandardizedString(jsonArray, JsonStandard.WRAP_NULL_STRINGS);;
|
||||
assertFalse("Wrapping null strings standard's broken", Objects.equals(jsonArray.toString(), resultingString));
|
||||
assertTrue(resultingString.contains("\"null\""));
|
||||
}
|
||||
|
||||
public void testNullStringsOnArrayUnwrapped() {
|
||||
JSONArray jsonArray = new JSONArray();
|
||||
jsonArray.add("abc");
|
||||
jsonArray.add("null");
|
||||
jsonArray.add(null);
|
||||
|
||||
String resultingString = JSONUtils.jsonToStandardizedString(jsonArray, JsonStandard.LEGACY);
|
||||
assertEquals(jsonArray.toString(), resultingString);
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
<groupId>org.neo4j.talend</groupId>
|
||||
<artifactId>neo4j-talend-component</artifactId>
|
||||
<version>1.3-20171206</version>
|
||||
<version>1.3-20191012</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>Neo4j Talend </name>
|
||||
@@ -100,12 +100,23 @@
|
||||
<artifactId>neo4j-import-tool</artifactId>
|
||||
<version>${neo4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.25</version>
|
||||
</dependency>
|
||||
<!-- Talend log4j library -->
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<version>1.7.25</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.16</version>
|
||||
<scope>provided</scope>
|
||||
<version>1.2.17</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- JUnit -->
|
||||
<dependency>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package org.neo4j.talend;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.neo4j.graphdb.DynamicLabel;
|
||||
import org.neo4j.graphdb.index.IndexHits;
|
||||
import org.neo4j.helpers.collection.MapUtil;
|
||||
@@ -23,7 +24,7 @@ public class Neo4jBatchDatabase {
|
||||
/**
|
||||
* The logger
|
||||
*/
|
||||
private static Logger log = Logger.getLogger(Neo4jBatchDatabase.class);
|
||||
private static Logger log = LoggerFactory.getLogger(Neo4jBatchDatabase.class);
|
||||
|
||||
/**
|
||||
* Name of the field index for the importId
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
package org.neo4j.talend;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.neo4j.graphdb.DynamicRelationshipType;
|
||||
|
||||
@@ -11,7 +12,7 @@ import java.util.Map;
|
||||
|
||||
public class Neo4jBatchInserterRelationship extends Neo4jBatchInserterAbstract {
|
||||
|
||||
private static Logger log = Logger.getLogger(Neo4jBatchInserterRelationship.class);
|
||||
private static Logger log = LoggerFactory.getLogger(Neo4jBatchInserterRelationship.class);
|
||||
|
||||
private String relationshipTypeField;
|
||||
private String direction;
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
package org.neo4j.talend;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.neo4j.tooling.ImportTool;
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
@@ -18,7 +19,7 @@ public class Neo4jImportTool {
|
||||
/**
|
||||
* The logger
|
||||
*/
|
||||
private final static Logger log = Logger.getLogger(Neo4jImportTool.class);
|
||||
private final static Logger log = LoggerFactory.getLogger(Neo4jImportTool.class);
|
||||
|
||||
protected final static String HEADERS_KEY = "HEADERS";
|
||||
protected final static String FILE_KEY = "FILE";
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>simpleexcel-2.1-20190507</artifactId>
|
||||
<artifactId>simpleexcel-2.2-20190722</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
|
||||
@@ -12,17 +12,21 @@
|
||||
// ============================================================================
|
||||
package com.talend.excel.xssf.event;
|
||||
|
||||
import org.apache.poi.ooxml.util.PackageHelper;
|
||||
import org.apache.poi.openxml4j.opc.OPCPackage;
|
||||
import org.apache.poi.poifs.crypt.Decryptor;
|
||||
import org.apache.poi.poifs.crypt.EncryptionInfo;
|
||||
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
|
||||
import org.apache.poi.xssf.eventusermodel.XSSFReader;
|
||||
import org.apache.poi.xssf.model.StylesTable;
|
||||
import org.apache.poi.ooxml.util.PackageHelper;
|
||||
import org.xml.sax.ContentHandler;
|
||||
import org.xml.sax.InputSource;
|
||||
import org.xml.sax.XMLReader;
|
||||
import org.xml.sax.helpers.XMLReaderFactory;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
@@ -32,7 +36,6 @@ import java.util.concurrent.FutureTask;
|
||||
|
||||
/**
|
||||
* created by wwang on 2012-9-27 Detailled comment
|
||||
*
|
||||
*/
|
||||
public class ExcelReader implements Callable {
|
||||
|
||||
@@ -46,6 +49,8 @@ public class ExcelReader implements Callable {
|
||||
|
||||
private String charset = "UTF-8";
|
||||
|
||||
private String password = null;
|
||||
|
||||
private java.io.InputStream is;
|
||||
|
||||
private List<String> sheetNames = new ArrayList<String>();
|
||||
@@ -64,15 +69,17 @@ public class ExcelReader implements Callable {
|
||||
task = new Thread(futureTask);
|
||||
}
|
||||
|
||||
public void parse(String fileURL, String charset) {
|
||||
public void parse(String fileURL, String charset, String password) {
|
||||
this.fileURL = fileURL;
|
||||
this.charset = charset;
|
||||
this.password = password;
|
||||
task.start();
|
||||
}
|
||||
|
||||
public void parse(java.io.InputStream is, String charset) {
|
||||
public void parse(java.io.InputStream is, String charset, String password) {
|
||||
this.is = is;
|
||||
this.charset = charset;
|
||||
this.password = password;
|
||||
task.start();
|
||||
}
|
||||
|
||||
@@ -120,11 +127,25 @@ public class ExcelReader implements Callable {
|
||||
|
||||
public Object call() throws Exception {
|
||||
OPCPackage pkg = null;
|
||||
POIFSFileSystem fs = null;
|
||||
try {
|
||||
if (fileURL != null) {
|
||||
pkg = OPCPackage.open(fileURL);
|
||||
if (password != null) {
|
||||
if (fileURL != null) {
|
||||
fs = new POIFSFileSystem(new File(fileURL));
|
||||
} else {
|
||||
fs = new POIFSFileSystem(is);
|
||||
}
|
||||
Decryptor d = Decryptor.getInstance(new EncryptionInfo(fs));
|
||||
if (!d.verifyPassword(password)) {
|
||||
throw new RuntimeException("Error: Cannot decrypt Excel file. Invalid password.");
|
||||
}
|
||||
pkg = OPCPackage.open(d.getDataStream(fs));
|
||||
} else {
|
||||
pkg = PackageHelper.open(is);
|
||||
if (fileURL != null) {
|
||||
pkg = OPCPackage.open(fileURL);
|
||||
} else {
|
||||
pkg = PackageHelper.open(is);
|
||||
}
|
||||
}
|
||||
XSSFReader r = new XSSFReader(pkg);
|
||||
|
||||
@@ -204,6 +225,9 @@ public class ExcelReader implements Callable {
|
||||
if (pkg != null) {
|
||||
pkg.revert();
|
||||
}
|
||||
if (fs != null) {
|
||||
fs.close();
|
||||
}
|
||||
cache.notifyErrorOccurred();
|
||||
}
|
||||
return null;
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend-soap</artifactId>
|
||||
<version>2.1-20190513</version>
|
||||
<version>2.1-20190716</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend-soap</name>
|
||||
|
||||
@@ -124,7 +124,9 @@ public class SOAPUtil {
|
||||
}
|
||||
SOAPMessage message = messageFactory.createMessage();
|
||||
MimeHeaders mimeHeaders = message.getMimeHeaders();
|
||||
mimeHeaders.setHeader("SOAPAction", soapAction);
|
||||
|
||||
setSoapAction(version, soapAction, mimeHeaders);
|
||||
|
||||
if (basicAuth) {
|
||||
addBasicAuthHeader(mimeHeaders, username, password);
|
||||
}
|
||||
@@ -263,8 +265,8 @@ public class SOAPUtil {
|
||||
}
|
||||
SOAPMessage message = messageFactory.createMessage();
|
||||
MimeHeaders mimeHeaders = message.getMimeHeaders();
|
||||
mimeHeaders.setHeader("SOAPAction", soapAction);
|
||||
SOAPPart soapPart = message.getSOAPPart();
|
||||
setSoapAction(version, soapAction, mimeHeaders);
|
||||
SOAPPart soapPart = message.getSOAPPart();
|
||||
|
||||
String encoding = getEncoding(soapMessage);
|
||||
|
||||
@@ -291,8 +293,20 @@ public class SOAPUtil {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String getEncoding(String text) {
|
||||
|
||||
/* https://jira.talendforge.org/browse/TDI-42581 skip add SOAPAction directly to header v1.2 */
|
||||
private void setSoapAction(String version, String soapAction, MimeHeaders mimeHeaders) {
|
||||
if (SOAP12.equals(version)) {
|
||||
// in soap version 1.2 param 'action' optional and should not be empty
|
||||
if( soapAction != null && !soapAction.trim().isEmpty()) {
|
||||
mimeHeaders.setHeader("Content-Type", "application/soap+xml; charset=utf-8; action=\"" + soapAction + "\"");
|
||||
}
|
||||
} else {
|
||||
mimeHeaders.setHeader("SOAPAction", soapAction);
|
||||
}
|
||||
}
|
||||
|
||||
private String getEncoding(String text) {
|
||||
String result = Charset.defaultCharset().name();
|
||||
|
||||
if(text == null) {
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend-codegen-utils</artifactId>
|
||||
<!-- release for revert version of library -->
|
||||
<version>0.25.4</version>
|
||||
<version>0.28.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
|
||||
@@ -51,6 +51,8 @@ public final class TypeConverter {
|
||||
|
||||
public static final String LIST = "id_List";
|
||||
|
||||
public static final String OBJECT = "id_Object";
|
||||
|
||||
private TypeConverter() {
|
||||
// Class provides static utility methods and shouldn't be instantiated
|
||||
}
|
||||
@@ -167,6 +169,8 @@ public final class TypeConverter {
|
||||
return SHORT;
|
||||
case "java.util.Date":
|
||||
return DATE;
|
||||
case "java.lang.Object":
|
||||
return OBJECT;
|
||||
default:
|
||||
throw new UnsupportedOperationException("Unrecognized java class " + javaClass);
|
||||
}
|
||||
|
||||
@@ -5,9 +5,9 @@
|
||||
>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>talend-db-exasol</artifactId>
|
||||
<version>2.1.2</version>
|
||||
<version>2.1.5</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend-db-exasol</name>
|
||||
@@ -33,10 +33,22 @@
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.25</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<version>1.7.25</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
|
||||
@@ -27,15 +27,15 @@ import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.log4j.Level;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
/**
|
||||
* This class provides an IMPORT command for the EXASol database.
|
||||
* @author Jan Lolling, jan.lolling@cimt-ag.de
|
||||
*/
|
||||
public class EXABulkUtil {
|
||||
|
||||
private static Logger logger = Logger.getLogger(EXABulkUtil.class);
|
||||
private static Logger logger = LoggerFactory.getLogger(EXABulkUtil.class);
|
||||
public static final String CSV = "CSV";
|
||||
public static final String FBV = "FBV";
|
||||
public static final String ORA = "ORA";
|
||||
@@ -83,23 +83,14 @@ public class EXABulkUtil {
|
||||
private int sourceIdentifierCase = 0; // 0 = unchanged, 1 = lower case, 2 = upper case
|
||||
private boolean onlyBuildSQLCode = false;
|
||||
|
||||
public void setDebug(boolean debug) {
|
||||
if (debug) {
|
||||
logger.setLevel(Level.DEBUG);
|
||||
} else {
|
||||
logger.setLevel(Level.INFO);
|
||||
}
|
||||
}
|
||||
|
||||
private String createNumberFormat(Integer length, Integer precision, boolean hasGroups) {
|
||||
if (length != null && length.intValue() > 0) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int numGroups = (length.intValue() / 3) + 1;
|
||||
for (int i = 0; i < numGroups; i++) {
|
||||
if (i > 0 && hasGroups) {
|
||||
sb.append("G");
|
||||
}
|
||||
sb.append("999");
|
||||
for (int i = length - 1; i >= 0; i--) {
|
||||
if(hasGroups && i < length - 1 && i > 0 && (i % 3 == 2)) {
|
||||
sb.append("G");
|
||||
}
|
||||
sb.append("9");
|
||||
}
|
||||
if (precision != null && precision.intValue() > 0) {
|
||||
sb.append("D");
|
||||
|
||||
@@ -39,7 +39,6 @@ public class TestEXABulkUtil {
|
||||
truncateStat.execute("truncate table " + errorTable);
|
||||
truncateStat.close();
|
||||
EXABulkUtil u = new EXABulkUtil();
|
||||
u.setDebug(true);
|
||||
u.setTransferSecure(true);
|
||||
u.setConnection(connection);
|
||||
u.setTable(table);
|
||||
@@ -83,7 +82,6 @@ public class TestEXABulkUtil {
|
||||
}
|
||||
truncateStat.close();
|
||||
EXABulkUtil u = new EXABulkUtil();
|
||||
u.setDebug(true);
|
||||
u.setConnection(connection);
|
||||
u.setTable(table);
|
||||
u.setRemoteFileUrl("sftp://172.16.214.132/home/tisadmin/Downloads/");
|
||||
@@ -130,7 +128,6 @@ public class TestEXABulkUtil {
|
||||
}
|
||||
truncateStat.close();
|
||||
EXABulkUtil u = new EXABulkUtil();
|
||||
u.setDebug(true);
|
||||
u.setConnection(connection);
|
||||
u.setTable(table);
|
||||
u.setDbmsSourceType("JDBC");
|
||||
@@ -170,7 +167,6 @@ public class TestEXABulkUtil {
|
||||
}
|
||||
truncateStat.close();
|
||||
EXABulkUtil u = new EXABulkUtil();
|
||||
u.setDebug(true);
|
||||
u.setConnection(connection);
|
||||
u.setTable(table);
|
||||
u.setDbmsSourceType("ORA");
|
||||
@@ -211,7 +207,6 @@ public class TestEXABulkUtil {
|
||||
}
|
||||
stat.close();
|
||||
EXABulkUtil u = new EXABulkUtil();
|
||||
u.setDebug(true);
|
||||
u.setConnection(connection);
|
||||
u.setTable(table);
|
||||
u.setDbmsSourceType("JDBC");
|
||||
@@ -250,7 +245,6 @@ public class TestEXABulkUtil {
|
||||
}
|
||||
stat.close();
|
||||
EXABulkUtil u = new EXABulkUtil();
|
||||
u.setDebug(true);
|
||||
u.setConnection(connection);
|
||||
u.setTable(table);
|
||||
u.setDbmsSourceType("JDBC");
|
||||
@@ -290,7 +284,6 @@ public class TestEXABulkUtil {
|
||||
}
|
||||
stat.close();
|
||||
EXABulkUtil u = new EXABulkUtil();
|
||||
u.setDebug(true);
|
||||
u.setConnection(connection);
|
||||
u.setTable(table);
|
||||
u.setDbmsSourceType("EXA");
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<groupId>org.talend</groupId>
|
||||
<artifactId>talend-httputil</artifactId>
|
||||
<name>talend-httputil</name>
|
||||
<version>1.0.4</version>
|
||||
<version>1.0.5</version>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
@@ -20,7 +20,7 @@
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>2.9.9</version>
|
||||
<version>2.10.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>job-audit</artifactId>
|
||||
<version>1.0</version>
|
||||
<version>1.1</version>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
@@ -50,28 +50,43 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.talend.daikon</groupId>
|
||||
<artifactId>daikon-audit</artifactId>
|
||||
<version>0.31.8</version>
|
||||
<artifactId>audit-common</artifactId>
|
||||
<version>1.8.0</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<!--
|
||||
<dependency>
|
||||
<groupId>org.talend.daikon</groupId>
|
||||
<artifactId>audit-logback</artifactId>
|
||||
<version>0.31.8</version>
|
||||
<artifactId>audit-log4j1</artifactId>
|
||||
<version>1.8.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
<version>1.3.0-alpha4</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
-->
|
||||
|
||||
<dependency>
|
||||
<groupId>org.talend.daikon</groupId>
|
||||
<artifactId>audit-log4j2</artifactId>
|
||||
<version>1.8.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-api</artifactId>
|
||||
<version>2.12.1</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
<version>2.12.1</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -18,13 +18,30 @@ public class JobEventAuditLoggerFactory {
|
||||
final AuditConfigurationMap config = AuditConfiguration.loadFromProperties(properties);
|
||||
|
||||
AbstractBackend logger = null;
|
||||
String loggerClass = "org.talend.logging.audit.logback.LogbackBackend";
|
||||
|
||||
//load log4j2 implement firstly
|
||||
String loggerClass = "org.talend.logging.audit.log4j2.Log4j2Backend";
|
||||
try {
|
||||
final Class<?> clz = Class.forName(loggerClass);
|
||||
logger = (AbstractBackend) clz.getConstructor(AuditConfigurationMap.class).newInstance(config);
|
||||
} catch (ReflectiveOperationException e) {
|
||||
throw new RuntimeException("Unable to load backend " + loggerClass, e);
|
||||
}
|
||||
final Class<?> clz = Class.forName(loggerClass);
|
||||
logger = (AbstractBackend) clz.getConstructor(AuditConfigurationMap.class).newInstance(config);
|
||||
} catch (ReflectiveOperationException e) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
//load log4j1 implement if not found log4j2
|
||||
if (logger == null) {
|
||||
loggerClass = "org.talend.logging.audit.log4j1.Log4j1Backend";
|
||||
try {
|
||||
final Class<?> clz = Class.forName(loggerClass);
|
||||
logger = (AbstractBackend) clz.getConstructor(AuditConfigurationMap.class).newInstance(config);
|
||||
} catch (ReflectiveOperationException e) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
|
||||
if(logger == null) {
|
||||
throw new RuntimeException("Unable to load backend : " + loggerClass);
|
||||
}
|
||||
|
||||
final DefaultAuditLoggerBase loggerBase = new DefaultAuditLoggerBase(logger, config);
|
||||
|
||||
|
||||
@@ -8,7 +8,8 @@ public class JobAuditLoggerTest {
|
||||
|
||||
public static void main(String[] args) {
|
||||
Properties props = new Properties();
|
||||
props.setProperty("root.logger", "audit");
|
||||
String root_logger_name = "audit";
|
||||
props.setProperty("root.logger", root_logger_name);
|
||||
props.setProperty("encoding", "UTF-8");
|
||||
props.setProperty("application.name", "Talend Studio");
|
||||
props.setProperty("service.name", "Talend Studio Job");
|
||||
@@ -19,6 +20,10 @@ public class JobAuditLoggerTest {
|
||||
props.setProperty("appender.file.maxsize", "52428800");
|
||||
props.setProperty("appender.file.maxbackup", "20");
|
||||
props.setProperty("host", "false");
|
||||
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel(root_logger_name, org.apache.logging.log4j.Level.DEBUG);
|
||||
//org.apache.log4j.Logger.getLogger("audit").setLevel(org.apache.log4j.Level.DEBUG);
|
||||
|
||||
final JobAuditLogger logger = JobEventAuditLoggerFactory.createJobAuditLogger(props);
|
||||
Context context = JobContextBuilder.create().jobName("fetch_from_s3_every_day").jobId("jobid_123")
|
||||
.jobVersion("0.1").connectorType("tXMLMAP").connectorId("tXMLMap_1")
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend-mscrm</artifactId>
|
||||
<version>3.4-20190513</version>
|
||||
<version>3.4-20191012</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend-mscrm</name>
|
||||
@@ -109,7 +109,7 @@
|
||||
<dependency>
|
||||
<groupId>com.microsoft.azure</groupId>
|
||||
<artifactId>adal4j</artifactId>
|
||||
<version>1.1.1-patch</version>
|
||||
<version>1.1.1-20191012</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
|
||||
@@ -22,7 +22,8 @@ import org.apache.axis2.transport.http.HTTPConstants;
|
||||
import org.apache.axis2.transport.http.HTTPTransportConstants;
|
||||
import org.apache.axis2.transport.http.HttpTransportProperties;
|
||||
import org.apache.axis2.transport.http.HttpTransportProperties.ProxyProperties;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.datacontract.schemas._2004._07.system_collections_generic.KeyValuePairOfEndpointTypestringztYlk6OT;
|
||||
import org.talend.ms.crm.sdk.OnlineAuthenticationPolicy;
|
||||
import org.talend.ms.crm.sdk.OrganizationServiceStubWrapper;
|
||||
@@ -61,7 +62,7 @@ import com.microsoft.schemas.xrm._2011.contracts.discovery.RetrieveOrganizationR
|
||||
*/
|
||||
public class MSCRMClient {
|
||||
|
||||
static Logger logger = Logger.getLogger(MSCRMClient.class.getName());
|
||||
static Logger logger = LoggerFactory.getLogger(MSCRMClient.class.getName());
|
||||
|
||||
/**
|
||||
* Microsoft account (e.g. youremail@live.com) or Microsoft Office 365 (Org ID e.g.
|
||||
|
||||
@@ -38,7 +38,8 @@ import org.apache.http.impl.client.SystemDefaultHttpClient;
|
||||
import org.apache.http.params.BasicHttpParams;
|
||||
import org.apache.http.params.CoreConnectionPNames;
|
||||
import org.apache.http.params.HttpParams;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.NamedNodeMap;
|
||||
import org.w3c.dom.Node;
|
||||
@@ -53,7 +54,7 @@ public final class DeviceIdManager {
|
||||
/**
|
||||
* Logger
|
||||
*/
|
||||
static final Logger Log = Logger.getLogger(DeviceIdManager.class.getName());
|
||||
static final Logger Log = LoggerFactory.getLogger(DeviceIdManager.class.getName());
|
||||
|
||||
private static final Random RandomInstance = new Random();
|
||||
|
||||
|
||||
@@ -16,7 +16,8 @@ import javax.wsdl.WSDLException;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.xpath.XPathExpressionException;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
/**
|
||||
@@ -25,7 +26,7 @@ import org.xml.sax.SAXException;
|
||||
*/
|
||||
public final class OnlineAuthenticationPolicy {
|
||||
|
||||
static Logger logger = Logger.getLogger(OnlineAuthenticationPolicy.class.getName());
|
||||
static Logger logger = LoggerFactory.getLogger(OnlineAuthenticationPolicy.class.getName());
|
||||
|
||||
/**
|
||||
* Construct an Instance of the OnlineAuthenticationPolicy class.
|
||||
|
||||
@@ -44,7 +44,8 @@ import org.apache.http.params.BasicHttpParams;
|
||||
import org.apache.http.params.CoreConnectionPNames;
|
||||
import org.apache.http.params.HttpParams;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import org.xml.sax.SAXException;
|
||||
@@ -55,7 +56,7 @@ import org.xml.sax.SAXException;
|
||||
*/
|
||||
public final class WsdlTokenManager {
|
||||
|
||||
static Logger logger = Logger.getLogger(WsdlTokenManager.class.getName());
|
||||
static Logger logger = LoggerFactory.getLogger(WsdlTokenManager.class.getName());
|
||||
|
||||
private final String DeviceTokenTemplate = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + " <s:Envelope "
|
||||
+ " xmlns:s=\"http://www.w3.org/2003/05/soap-envelope\""
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>talend-proxy</artifactId>
|
||||
<version>1.0.0</version>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<source>8</source>
|
||||
<target>8</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
|
||||
<snapshots><enabled>true</enabled></snapshots>
|
||||
<releases><enabled>false</enabled></releases>
|
||||
</snapshotRepository>
|
||||
<repository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
<snapshots><enabled>false</enabled></snapshots>
|
||||
<releases><enabled>true</enabled></releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
<version>3.9</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,20 @@
|
||||
package org.talend.proxy;
|
||||
|
||||
import org.apache.commons.lang3.tuple.ImmutablePair;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
|
||||
public class ProxyCreds {
|
||||
private Pair<String, String> credPair;
|
||||
|
||||
public ProxyCreds(String userName, String pass) {
|
||||
this.credPair = new ImmutablePair<>(userName, pass);
|
||||
}
|
||||
|
||||
public String getUser() {
|
||||
return credPair.getKey();
|
||||
}
|
||||
|
||||
public String getPass() {
|
||||
return credPair.getValue();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package org.talend.proxy;
|
||||
|
||||
import java.net.Proxy;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
/**
|
||||
* Use only inside of ThreadLocal
|
||||
*/
|
||||
public class ProxyHolder {
|
||||
private Map<String, Proxy> proxyMap;
|
||||
|
||||
public ProxyHolder() {
|
||||
proxyMap = new HashMap<>();
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param proxy HTTP or SOCKS proxy instance to use
|
||||
* @param host without protocol
|
||||
* @param port -1 to apply proxy for every port
|
||||
*/
|
||||
public void putNewHost(Proxy proxy, String host, int port) {
|
||||
if (port != -1) {
|
||||
proxyMap.put(host + ":" + port, proxy);
|
||||
} else {
|
||||
proxyMap.put(host, proxy);
|
||||
}
|
||||
}
|
||||
|
||||
public Map<String, Proxy> getProxyMap() {
|
||||
return proxyMap;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package org.talend.proxy;
|
||||
|
||||
import java.net.Authenticator;
|
||||
import java.net.PasswordAuthentication;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class TalendProxyAuthenticator extends Authenticator {
|
||||
private static TalendProxyAuthenticator instance;
|
||||
|
||||
public static synchronized TalendProxyAuthenticator getInstance() {
|
||||
if (instance == null) {
|
||||
instance = new TalendProxyAuthenticator();
|
||||
}
|
||||
|
||||
return instance;
|
||||
}
|
||||
|
||||
private TalendProxyAuthenticator() {
|
||||
}
|
||||
|
||||
private Map<String, ProxyCreds> proxyCredsMap = new HashMap<>();
|
||||
|
||||
public synchronized void addAuthForProxy(String host, String port, String userName, String pass) {
|
||||
proxyCredsMap.put(host + ":" + port, new ProxyCreds(userName, pass));
|
||||
}
|
||||
|
||||
public synchronized ProxyCreds getCredsForProxyURI(String proxyURI) {
|
||||
return proxyCredsMap.get(proxyURI);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected synchronized PasswordAuthentication getPasswordAuthentication() {
|
||||
String requestURI = super.getRequestingHost() + ":" + super.getRequestingPort();
|
||||
if (proxyCredsMap.containsKey(requestURI)) {
|
||||
return new PasswordAuthentication(proxyCredsMap.get(requestURI).getUser(), proxyCredsMap.get(requestURI).getPass().toCharArray());
|
||||
} else {
|
||||
return super.getPasswordAuthentication(); //don't use authentication
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package org.talend.proxy;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.Proxy;
|
||||
import java.net.ProxySelector;
|
||||
import java.net.SocketAddress;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
|
||||
public class TalendProxySelector extends ProxySelector {
|
||||
private static TalendProxySelector instance;
|
||||
|
||||
private static final Logger log = Logger.getLogger(TalendProxySelector.class);
|
||||
|
||||
private ThreadLocal<ProxyHolder> threadLocalProxyHolder;
|
||||
private ProxyHolder globalProxyHolder;
|
||||
|
||||
public static synchronized TalendProxySelector getInstance() {
|
||||
if (instance == null) {
|
||||
instance = new TalendProxySelector();
|
||||
}
|
||||
|
||||
return instance;
|
||||
}
|
||||
|
||||
private TalendProxySelector() {
|
||||
globalProxyHolder = new ProxyHolder();
|
||||
}
|
||||
|
||||
public synchronized void addProxySettings(Proxy proxy, boolean threadSpecific, String host, int port) {
|
||||
if (threadSpecific) {
|
||||
if (threadLocalProxyHolder == null) {
|
||||
threadLocalProxyHolder = new ThreadLocal<>();
|
||||
}
|
||||
if (threadLocalProxyHolder.get() == null) {
|
||||
ProxyHolder newProxyHolder = new ProxyHolder();
|
||||
threadLocalProxyHolder.set(newProxyHolder);
|
||||
}
|
||||
|
||||
threadLocalProxyHolder.get().putNewHost(proxy, host, port);
|
||||
} else {
|
||||
globalProxyHolder.putNewHost(proxy, host, port);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds and return proxy was set for specific host
|
||||
* @param uriString host:port
|
||||
* @return Optional of Proxy if such proxy setting was set
|
||||
*/
|
||||
public synchronized Proxy getProxyForUriString(String uriString) {
|
||||
if (proxyHolderContainsHost(globalProxyHolder, uriString)) {
|
||||
log.debug("All threads proxy " + globalProxyHolder.getProxyMap().get(uriString) + " is using to connect to URI " + uriString);
|
||||
return globalProxyHolder.getProxyMap().containsKey(uriString) ? globalProxyHolder.getProxyMap().get(uriString) :
|
||||
globalProxyHolder.getProxyMap().get(uriString.substring(0, uriString.lastIndexOf(":")));
|
||||
} else if (threadLocalProxyHolder != null && proxyHolderContainsHost(threadLocalProxyHolder.get(), uriString)) {
|
||||
log.debug("Proxy " + threadLocalProxyHolder.get().getProxyMap().get(uriString) + " is using to connect to URI " + uriString);
|
||||
return threadLocalProxyHolder.get().getProxyMap().containsKey(uriString) ?
|
||||
threadLocalProxyHolder.get().getProxyMap().get(uriString) :
|
||||
threadLocalProxyHolder.get().getProxyMap().get(uriString.substring(0, uriString.lastIndexOf(":")));
|
||||
} else {
|
||||
log.debug("No proxy is using to connect to URI " + uriString);
|
||||
return Proxy.NO_PROXY;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Proxy> select(URI uri) {
|
||||
String uriString = uri.getHost();
|
||||
if (uri.getPort() != -1) {
|
||||
uriString += ":" + uri.getPort() ;
|
||||
}
|
||||
log.debug("Network request hadling from Talend proxy selector. Thread " + Thread.currentThread().getName() + ". URI to connect: " + uriString);
|
||||
return Collections.singletonList(getProxyForUriString(uriString));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void connectFailed(URI uri, SocketAddress sa, IOException ioe) {
|
||||
if (ioe != null) {
|
||||
log.warn("Connect failed when use Talend ProxySelector to the URI:" + uri.toString(), ioe);
|
||||
} else {
|
||||
log.warn("Connect failed when use Talend ProxySelector to the " + uri);
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean proxyHolderContainsHost(ProxyHolder holder, String uriString) {
|
||||
return holder != null &&
|
||||
(holder.getProxyMap().containsKey(uriString)
|
||||
|| (uriString.contains(":") && holder.getProxyMap().containsKey(uriString.substring(0, uriString.indexOf(":")))));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,12 +4,12 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend-ws</artifactId>
|
||||
<version>1.0.1-20190204</version>
|
||||
<version>1.0.1-20191112</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<cxf.version>3.1.1</cxf.version>
|
||||
<cxf.version>3.3.4</cxf.version>
|
||||
<odata.version>4.3.0</odata.version>
|
||||
<slf4j.version>1.7.12</slf4j.version>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
@@ -90,42 +90,42 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.cxf</groupId>
|
||||
<artifactId>cxf-core</artifactId>
|
||||
<version>3.2.6</version>
|
||||
<version>${cxf.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.cxf</groupId>
|
||||
<artifactId>cxf-rt-frontend-simple</artifactId>
|
||||
<version>3.2.6</version>
|
||||
<version>${cxf.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.cxf</groupId>
|
||||
<artifactId>cxf-tools-common</artifactId>
|
||||
<version>3.2.6</version>
|
||||
<version>${cxf.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.cxf</groupId>
|
||||
<artifactId>cxf-rt-wsdl</artifactId>
|
||||
<version>3.2.6</version>
|
||||
<version>${cxf.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.cxf</groupId>
|
||||
<artifactId>cxf-rt-bindings-soap</artifactId>
|
||||
<version>3.2.6</version>
|
||||
<version>${cxf.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.cxf</groupId>
|
||||
<artifactId>cxf-rt-databinding-jaxb</artifactId>
|
||||
<version>3.2.6</version>
|
||||
<version>${cxf.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.cxf</groupId>
|
||||
<artifactId>cxf-rt-transports-http-jetty</artifactId>
|
||||
<version>3.2.6</version>
|
||||
<version>${cxf.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.cxf</groupId>
|
||||
<artifactId>cxf-rt-transports-http</artifactId>
|
||||
<version>3.2.6</version>
|
||||
<version>${cxf.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talendExcel-1.4-20190531</artifactId>
|
||||
<artifactId>talendExcel-1.5-20190731</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package org.talend;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.Date;
|
||||
@@ -10,17 +11,23 @@ import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.poi.poifs.crypt.EncryptionInfo;
|
||||
import org.apache.poi.poifs.crypt.EncryptionMode;
|
||||
import org.apache.poi.poifs.crypt.Encryptor;
|
||||
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.CellStyle;
|
||||
import org.apache.poi.ss.usermodel.CellType;
|
||||
import org.apache.poi.ss.usermodel.Font;
|
||||
import org.apache.poi.ss.usermodel.FormulaEvaluator;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.apache.poi.xssf.streaming.SXSSFSheet;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.apache.poi.ss.usermodel.FormulaEvaluator;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbookType;
|
||||
|
||||
public class ExcelTool {
|
||||
|
||||
@@ -70,6 +77,8 @@ public class ExcelTool {
|
||||
|
||||
private boolean isTrackAllColumns = false;
|
||||
|
||||
private String password = null;
|
||||
|
||||
public ExcelTool() {
|
||||
cellStylesMapping = new HashMap<>();
|
||||
}
|
||||
@@ -110,21 +119,7 @@ public class ExcelTool {
|
||||
initPreXlsx(fileName);
|
||||
}
|
||||
if (appendWorkbook) {
|
||||
InputStream inp = new FileInputStream(fileName);
|
||||
wb = WorkbookFactory.create(inp);
|
||||
sheet = wb.getSheet(sheetName);
|
||||
if (sheet != null) {
|
||||
if (appendSheet) {
|
||||
if (sheet.getLastRowNum() != 0 || sheet.getRow(0) != null) {
|
||||
curY = sheet.getLastRowNum() + 1;
|
||||
}
|
||||
} else {
|
||||
wb.removeSheetAt(wb.getSheetIndex(sheetName));
|
||||
sheet = wb.createSheet(sheetName);
|
||||
}
|
||||
} else {
|
||||
sheet = wb.createSheet(sheetName);
|
||||
}
|
||||
appendActionForFile(fileName);
|
||||
} else {
|
||||
xlsxFile.delete();
|
||||
wb = new SXSSFWorkbook(rowAccessWindowSize);
|
||||
@@ -140,6 +135,47 @@ public class ExcelTool {
|
||||
}
|
||||
}
|
||||
|
||||
public void prepareXlsmFile(String fileName) throws Exception {
|
||||
File xlsmFile = new File(fileName);
|
||||
if (xlsmFile.exists()) {
|
||||
if (isAbsY && keepCellFormat) {
|
||||
initPreXlsx(fileName);
|
||||
}
|
||||
if (appendWorkbook) {
|
||||
appendActionForFile(fileName);
|
||||
} else {
|
||||
xlsmFile.delete();
|
||||
wb = new SXSSFWorkbook(new XSSFWorkbook(XSSFWorkbookType.XLSM), rowAccessWindowSize);
|
||||
sheet = wb.createSheet(sheetName);
|
||||
}
|
||||
} else {
|
||||
wb = new SXSSFWorkbook(new XSSFWorkbook(XSSFWorkbookType.XLSM), rowAccessWindowSize);
|
||||
sheet = wb.createSheet(sheetName);
|
||||
}
|
||||
if (isAbsY) {
|
||||
startX = absX;
|
||||
curY = absY;
|
||||
}
|
||||
}
|
||||
|
||||
private void appendActionForFile(String fileName) throws IOException {
|
||||
InputStream inp = new FileInputStream(fileName);
|
||||
wb = WorkbookFactory.create(inp);
|
||||
sheet = wb.getSheet(sheetName);
|
||||
if (sheet != null) {
|
||||
if (appendSheet) {
|
||||
if (sheet.getLastRowNum() != 0 || sheet.getRow(0) != null) {
|
||||
curY = sheet.getLastRowNum() + 1;
|
||||
}
|
||||
} else {
|
||||
wb.removeSheetAt(wb.getSheetIndex(sheetName));
|
||||
sheet = wb.createSheet(sheetName);
|
||||
}
|
||||
} else {
|
||||
sheet = wb.createSheet(sheetName);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @return start insert row index.
|
||||
@@ -302,13 +338,14 @@ public class ExcelTool {
|
||||
}
|
||||
|
||||
public void writeExcel(OutputStream outputStream) throws Exception {
|
||||
try {
|
||||
wb.write(outputStream);
|
||||
} finally {
|
||||
if(outputStream != null) {
|
||||
outputStream.close();
|
||||
}
|
||||
}
|
||||
try {
|
||||
wb.write(outputStream);
|
||||
wb.close();
|
||||
} finally {
|
||||
if (outputStream != null) {
|
||||
outputStream.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void writeExcel(String fileName, boolean createDir) throws Exception {
|
||||
@@ -319,14 +356,29 @@ public class ExcelTool {
|
||||
pFile.mkdirs();
|
||||
}
|
||||
}
|
||||
FileOutputStream fileOutput = new FileOutputStream(fileName);
|
||||
if (appendWorkbook && appendSheet && recalculateFormula) {
|
||||
evaluateFormulaCell();
|
||||
}
|
||||
FileOutputStream fileOutput = new FileOutputStream(fileName);
|
||||
POIFSFileSystem fs = null;
|
||||
try {
|
||||
wb.write(fileOutput);
|
||||
if (password == null) {
|
||||
wb.write(fileOutput);
|
||||
} else {
|
||||
fs = new POIFSFileSystem();
|
||||
Encryptor encryptor = new EncryptionInfo(EncryptionMode.agile).getEncryptor();
|
||||
encryptor.confirmPassword(password);
|
||||
OutputStream encryptedDataStream = encryptor.getDataStream(fs);
|
||||
wb.write(encryptedDataStream);
|
||||
encryptedDataStream.close(); // this is mandatory to do that at that point
|
||||
fs.writeFilesystem(fileOutput);
|
||||
}
|
||||
} finally {
|
||||
fileOutput.close();
|
||||
wb.close();
|
||||
fileOutput.close();
|
||||
if (fs != null) {
|
||||
fs.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -349,4 +401,9 @@ public class ExcelTool {
|
||||
((SXSSFSheet) sheet).flushRows();
|
||||
}
|
||||
}
|
||||
|
||||
public void setPasswordProtection(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talendMsgMailUtil-1.1-20170419</artifactId>
|
||||
<artifactId>talendMsgMailUtil-1.1-20191012</artifactId>
|
||||
<name>talendMsgMailUtil</name>
|
||||
<version>6.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
@@ -37,6 +37,11 @@
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.25</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi</artifactId>
|
||||
|
||||
@@ -12,8 +12,8 @@ import org.apache.poi.hsmf.datatypes.MAPIProperty;
|
||||
import org.apache.poi.hsmf.datatypes.StringChunk;
|
||||
import org.apache.poi.hsmf.datatypes.Types;
|
||||
import org.apache.poi.hsmf.exceptions.ChunkNotFoundException;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.apache.log4j.Level;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class MsgMailUtil {
|
||||
|
||||
@@ -24,6 +24,15 @@ public class MsgMailUtil {
|
||||
|
||||
public MsgMailUtil() {
|
||||
}
|
||||
|
||||
private enum Level {
|
||||
DEBUG,
|
||||
INFO,
|
||||
TRACE,
|
||||
FATAL,
|
||||
WARN,
|
||||
ERROR
|
||||
}
|
||||
|
||||
public MsgMailUtil(String fileName, String outAttachmentPath)
|
||||
throws IOException {
|
||||
@@ -31,8 +40,8 @@ public class MsgMailUtil {
|
||||
this.outAttachmentPath = outAttachmentPath;
|
||||
}
|
||||
|
||||
public void activeLog(Logger log, String position) {
|
||||
this.log = log;
|
||||
public void activeLog(String logger_name, String position) {
|
||||
this.log = LoggerFactory.getLogger(logger_name);
|
||||
this.position = position;
|
||||
}
|
||||
|
||||
@@ -113,23 +122,20 @@ public class MsgMailUtil {
|
||||
}
|
||||
}
|
||||
|
||||
public void processLog(Level level, String message) {
|
||||
private void processLog(Level level, String message) {
|
||||
message = position + " - " + message;
|
||||
if (this.log != null) {
|
||||
switch (level.toInt()) {
|
||||
case Level.TRACE_INT:
|
||||
switch (level) {
|
||||
case TRACE:
|
||||
log.trace(message);
|
||||
break;
|
||||
case Level.ERROR_INT:
|
||||
case ERROR:
|
||||
log.error(message);
|
||||
break;
|
||||
case Level.FATAL_INT:
|
||||
log.fatal(message);
|
||||
break;
|
||||
case Level.INFO_INT:
|
||||
case INFO:
|
||||
log.info(message);
|
||||
break;
|
||||
case Level.WARN_INT:
|
||||
case WARN:
|
||||
log.warn(message);
|
||||
break;
|
||||
default:
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend_DB_mssqlUtil-1.3-20190523</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>talend_DB_mssqlUtil</artifactId>
|
||||
<version>1.4</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend_DB_mssqlUtil</name>
|
||||
|
||||
@@ -16,6 +16,8 @@ import java.sql.SQLException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.TimeZone;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.text.ParseException;
|
||||
|
||||
/**
|
||||
* DOC Administrator class global comment. Detailled comment
|
||||
@@ -117,10 +119,15 @@ public class MSSqlGenerateTimestampUtil {
|
||||
return null;
|
||||
}
|
||||
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
|
||||
{
|
||||
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
|
||||
}
|
||||
|
||||
/**
|
||||
* parse datetimeoffset string to date.
|
||||
* datetimeoffset string show as YYYY-MM-DD hh:mm:ss[.nnnnnnn] [+|-]hh:mm
|
||||
* and no enough infomation for the DST decision,so no DST consider[not sure if 100% support for DST].
|
||||
*
|
||||
* @param datetimeOffsetString
|
||||
* @return
|
||||
@@ -131,13 +138,17 @@ public class MSSqlGenerateTimestampUtil {
|
||||
String offsetString = datetimeOffsetString.substring(idx + 1);
|
||||
int offset = TimeZone.getTimeZone("GMT" + offsetString).getRawOffset();
|
||||
|
||||
// get local timezone, also consider the DST
|
||||
TimeZone local = TimeZone.getDefault();
|
||||
int localOffset = local.getOffset(new java.util.Date().getTime());
|
||||
|
||||
long milliseconds = java.sql.Timestamp.valueOf(datetimeString).getTime();
|
||||
long millisecondsToGMT1970_01_01 = milliseconds - offset + localOffset;
|
||||
return new java.util.Date(millisecondsToGMT1970_01_01);
|
||||
long millisecondsWithOffset = 0l;
|
||||
try {
|
||||
millisecondsWithOffset = sdf.parse(datetimeString).getTime();
|
||||
} catch(ParseException e) {
|
||||
throw new RuntimeException(e.getMessage());
|
||||
}
|
||||
|
||||
//get GMT's millisecond number since GMT 1970_01_01
|
||||
long milliseconds4GMT_to_GMT_1970_01_01 = millisecondsWithOffset - offset;
|
||||
|
||||
return new java.util.Date(milliseconds4GMT_to_GMT_1970_01_01);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>talendsap</artifactId>
|
||||
<name>talendsap</name>
|
||||
<version>1.0.1</version>
|
||||
<version>1.0.2</version>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
|
||||
@@ -90,6 +90,7 @@ public class DocumentExtractor {
|
||||
List<Element> tablesAndChangingElements = new ArrayList<Element>(3);
|
||||
tablesAndChangingElements.add(functionElement.element("TABLES"));
|
||||
tablesAndChangingElements.add(functionElement.element("CHANGING"));
|
||||
tablesAndChangingElements.add(functionElement.element("OUTPUT"));
|
||||
|
||||
for(Element tablesOrChangingElement : tablesAndChangingElements) {
|
||||
if (tablesOrChangingElement == null) {
|
||||
@@ -121,6 +122,10 @@ public class DocumentExtractor {
|
||||
}
|
||||
result.add(row);
|
||||
}
|
||||
|
||||
if(!result.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
@@ -49,11 +49,11 @@ public class DocumentHelper {
|
||||
}
|
||||
}
|
||||
|
||||
public void addSingleParameter(String name, String value, boolean isChanging) {
|
||||
public void addSingleParameter(String name, String value, SAPParameterType parameter_type) {
|
||||
if(value == null) {
|
||||
value = "";
|
||||
}
|
||||
if (isChanging) {
|
||||
if (parameter_type == SAPParameterType.CHANGING) {
|
||||
correctChanging();
|
||||
changing.addElement(name).setText(value);
|
||||
} else {
|
||||
@@ -62,8 +62,8 @@ public class DocumentHelper {
|
||||
}
|
||||
}
|
||||
|
||||
public void addStructParameter(String name, boolean isChanging) {
|
||||
if (isChanging) {
|
||||
public void addStructParameter(String name, SAPParameterType parameter_type) {
|
||||
if (parameter_type == SAPParameterType.CHANGING) {
|
||||
correctChanging();
|
||||
currentStruct = changing.addElement(name);
|
||||
} else {
|
||||
@@ -79,13 +79,16 @@ public class DocumentHelper {
|
||||
currentStruct.addElement(name).setText(value);
|
||||
}
|
||||
|
||||
public void addTableParameter(String name, boolean isChanging) {
|
||||
if (isChanging) {
|
||||
public void addTableParameter(String name, SAPParameterType parameter_type) {
|
||||
if (parameter_type == SAPParameterType.CHANGING) {
|
||||
correctChanging();
|
||||
currentTable = changing.addElement(name);
|
||||
} else {
|
||||
} else if(parameter_type == SAPParameterType.TABLES) {
|
||||
correctTables();
|
||||
currentTable = tables.addElement(name);
|
||||
} else {
|
||||
correctInput();
|
||||
currentTable = input.addElement(name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,18 +111,18 @@ public class DocumentHelper {
|
||||
DocumentHelper helper = new DocumentHelper();
|
||||
helper.setFunctionName("READ_TABLE_FUNCTION");
|
||||
|
||||
helper.addSingleParameter("ID", "1", true);
|
||||
helper.addSingleParameter("NAME", "gaoyan", false);
|
||||
helper.addSingleParameter("ID", "1", SAPParameterType.CHANGING);
|
||||
helper.addSingleParameter("NAME", "gaoyan", SAPParameterType.IMPORT);
|
||||
|
||||
helper.addStructParameter("INFO", true);
|
||||
helper.addStructParameter("INFO", SAPParameterType.CHANGING);
|
||||
helper.addStructChildParameter("ID", "2");
|
||||
helper.addStructChildParameter("NAME", "wangwei");
|
||||
|
||||
helper.addStructParameter("INFO1", false);
|
||||
helper.addStructParameter("INFO1", SAPParameterType.IMPORT);
|
||||
helper.addStructChildParameter("ID1", "4");
|
||||
helper.addStructChildParameter("NAME1", "momo");
|
||||
|
||||
helper.addTableParameter("TABLE1", false);
|
||||
helper.addTableParameter("TABLE1", SAPParameterType.TABLES);
|
||||
for (int i = 0; i < 200000; i++) {
|
||||
helper.addTableRow();
|
||||
helper.addTableRowChildParameter("c1", i + "");
|
||||
@@ -132,7 +135,7 @@ public class DocumentHelper {
|
||||
helper.addTableRowChildParameter("c8", "wangwei" + i);
|
||||
}
|
||||
|
||||
helper.addTableParameter("TABLE2", false);
|
||||
helper.addTableParameter("TABLE2", SAPParameterType.TABLES);
|
||||
for (int i = 0; i < 2; i++) {
|
||||
helper.addTableRow();
|
||||
helper.addTableRowChildParameter("ID4", i + "");
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
package org.talend.sap;
|
||||
|
||||
public enum SAPParameterType {
|
||||
IMPORT,
|
||||
CHANGING,
|
||||
TABLES,
|
||||
EXPORT
|
||||
}
|
||||
@@ -4,7 +4,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talendzip</artifactId>
|
||||
<version>1.0-20190527</version>
|
||||
<version>1.0-20190917</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
@@ -40,19 +40,19 @@
|
||||
<dependency>
|
||||
<groupId>net.lingala.zip4j</groupId>
|
||||
<artifactId>zip4j</artifactId>
|
||||
<version>1.3.1</version>
|
||||
<version>1.3.3</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<version>1.10</version>
|
||||
<version>1.19</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>checkArchive-1.1-20181130</artifactId>
|
||||
<artifactId>checkArchive-1.1-20190917</artifactId>
|
||||
<version>6.0.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
</project>
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package com.talend.compress.zip;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class Util {
|
||||
public java.util.List<UnzippedFile> unzippedFiles = new java.util.ArrayList<UnzippedFile>();
|
||||
|
||||
@@ -53,6 +55,10 @@ public class Util {
|
||||
f = new java.io.File(path, shortName);
|
||||
}
|
||||
|
||||
if(!f.getCanonicalPath().startsWith(new java.io.File(path).getCanonicalPath())) {
|
||||
throw new IOException("expanding " + f.getName()
|
||||
+ " would create file outside of " + path);
|
||||
}
|
||||
checkDir(f);
|
||||
|
||||
f.getParentFile().mkdirs();
|
||||
@@ -89,6 +95,10 @@ public class Util {
|
||||
f = new java.io.File(path, shortName);
|
||||
}
|
||||
|
||||
if(!f.getCanonicalPath().startsWith(new java.io.File(path).getCanonicalPath())) {
|
||||
throw new IOException("expanding " + f.getName()
|
||||
+ " would create file outside of " + path);
|
||||
}
|
||||
if (isDirectory) {
|
||||
if (!f.exists()) {
|
||||
f.mkdirs();
|
||||
|
||||
@@ -5,9 +5,9 @@
|
||||
>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>thashfile</artifactId>
|
||||
<version>3.0-20170711</version>
|
||||
<version>3.1-20190910</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>thashfile</name>
|
||||
@@ -45,8 +45,8 @@
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.5.1</version>
|
||||
<configuration>
|
||||
<source>1.7</source>
|
||||
<target>1.7</target>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
|
||||
@@ -9,45 +9,44 @@ import java.util.concurrent.ConcurrentHashMap;
|
||||
import org.talend.designer.components.hashfile.memory.AdvancedMemoryHashFile;
|
||||
|
||||
public class MapHashFile {
|
||||
//use this map instead of globalMap
|
||||
// use this map instead of globalMap
|
||||
private Map<String, AdvancedMemoryHashFile> resourceMap = new ConcurrentHashMap<>();
|
||||
//keep the present key of AdvancedMemoryHashFile as key and the previous key as value
|
||||
// keep the present key of AdvancedMemoryHashFile as key and the previous key as
|
||||
// value
|
||||
private Map<String, String> keyMap = new ConcurrentHashMap<>();
|
||||
//singleton
|
||||
// singleton
|
||||
private static final MapHashFile mhf = new MapHashFile();
|
||||
|
||||
public static TalendMultiThreadLockMap resourceLockMap = new TalendMultiThreadLockMap();
|
||||
public static TalendMultiThreadLockMap resourceLockMap = new TalendMultiThreadLockMap();
|
||||
|
||||
public static class TalendMultiThreadLockMap {
|
||||
public static class TalendMultiThreadLockMap {
|
||||
|
||||
private Map<Object, Object> tMultiTheadLockMap = new HashMap<Object, Object>();
|
||||
private Map<Object, Object> tMultiTheadLockMap = new ConcurrentHashMap<>();
|
||||
|
||||
public Object get(Object key) {
|
||||
return tMultiTheadLockMap.computeIfAbsent(key, k -> new Object());
|
||||
}
|
||||
|
||||
public void remove(Object key) {
|
||||
tMultiTheadLockMap.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized Object get(Object key) {
|
||||
if (tMultiTheadLockMap.get(key) == null) {
|
||||
tMultiTheadLockMap.put(key, new Object());
|
||||
}
|
||||
return tMultiTheadLockMap.get(key);
|
||||
}
|
||||
|
||||
public synchronized void remove(Object key){
|
||||
tMultiTheadLockMap.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
private MapHashFile() {
|
||||
}
|
||||
|
||||
public static MapHashFile getMapHashFile() {
|
||||
return mhf;
|
||||
}
|
||||
|
||||
//get the linked AdvancedMemoryHashFile
|
||||
|
||||
// get the linked AdvancedMemoryHashFile
|
||||
public AdvancedMemoryHashFile getAdvancedMemoryHashFile(String key) {
|
||||
AdvancedMemoryHashFile amhf = resourceMap.get(key);
|
||||
String prekey = keyMap.get(key);
|
||||
//if present AdvancedMemoryHashFile is null get the AdvancedMemoryHashFile before present.
|
||||
// if present AdvancedMemoryHashFile is null get the AdvancedMemoryHashFile
|
||||
// before present.
|
||||
int size = keyMap.size();
|
||||
while(amhf==null && (size--)>0){
|
||||
while (amhf == null && (size--) > 0) {
|
||||
amhf = resourceMap.get(prekey);
|
||||
prekey = keyMap.get(prekey);
|
||||
}
|
||||
@@ -61,16 +60,18 @@ public class MapHashFile {
|
||||
public Map<String, String> getKeyMap() {
|
||||
return keyMap;
|
||||
}
|
||||
public void clearCache(String key){
|
||||
|
||||
public void clearCache(String key) {
|
||||
clearChildCache(getRootCache(key));
|
||||
}
|
||||
public void clearChildCache(String root){
|
||||
|
||||
public void clearChildCache(String root) {
|
||||
Set<String> set = keyMap.keySet();
|
||||
synchronized(keyMap) {
|
||||
synchronized (keyMap) {
|
||||
Iterator<String> it = set.iterator();
|
||||
while(it.hasNext()){
|
||||
while (it.hasNext()) {
|
||||
String key = it.next();
|
||||
if(root.equals(keyMap.get(key))){
|
||||
if (root.equals(keyMap.get(key))) {
|
||||
this.resourceMap.remove(key);
|
||||
this.keyMap.remove(key);
|
||||
clearChildCache(key);
|
||||
@@ -79,11 +80,11 @@ public class MapHashFile {
|
||||
}
|
||||
this.resourceMap.remove(root);
|
||||
}
|
||||
|
||||
public String getRootCache(String cache){
|
||||
|
||||
public String getRootCache(String cache) {
|
||||
String root;
|
||||
while((root = keyMap.get(cache))!=null){
|
||||
cache=root;
|
||||
while ((root = keyMap.get(cache)) != null) {
|
||||
cache = root;
|
||||
}
|
||||
return cache;
|
||||
}
|
||||
|
||||
@@ -149,6 +149,8 @@
|
||||
<IMPORT NAME="Driver-V5R3_V6R1" MODULE="jt400_V5R3.jar" MVN="mvn:org.talend.libraries/jt400_V5R3/6.0.0" REQUIRED_IF="DB_VERSION == 'jt400_V5R3.jar'" />
|
||||
<IMPORT NAME="Driver-V6R1_V7R2" MODULE="jt400_V6R1.jar" MVN="mvn:org.talend.libraries/jt400_V6R1/6.0.0" REQUIRED_IF="DB_VERSION == 'jt400_V6R1.jar'" />
|
||||
<IMPORT NAME="Driver-V7R1_V7R3" MODULE="jt400-9.8.jar" MVN="mvn:net.sf.jt400/jt400/9.8" REQUIRED_IF="DB_VERSION == 'jt400-9.8.jar'" />
|
||||
<IMPORT NAME="slf4j-api-1.7.25.jar" MODULE="slf4j-api-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-api/1.7.25" REQUIRED_IF="USE_SHARED_CONNECTION =='true'" />
|
||||
<IMPORT NAME="slf4j-log4j12-1.7.25.jar" MODULE="slf4j-log4j12-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-log4j12/1.7.25" REQUIRED_IF="USE_SHARED_CONNECTION =='true'" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -301,6 +301,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
pstmt_<%=cid %>.addBatch();
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
<%dbLog.data().addingToBatch(dbLog.var("nb_line"), dbLog.str(dataAction));%>
|
||||
batchSizeCounter_<%=cid%>++;
|
||||
<%
|
||||
@@ -308,6 +309,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.sql.SQLException e) {
|
||||
@@ -431,6 +433,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
pstmt_<%=cid %>.addBatch();
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
<%dbLog.data().addingToBatch(dbLog.var("nb_line"), dbLog.str(dataAction));%>
|
||||
batchSizeCounter_<%=cid%>++;
|
||||
<%
|
||||
@@ -438,6 +441,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.sql.SQLException e) {
|
||||
@@ -584,6 +588,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
@@ -659,6 +664,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
@@ -833,11 +839,13 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
}else{
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
}
|
||||
} catch(java.sql.SQLException e) {
|
||||
@@ -914,12 +922,14 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
pstmt_<%=cid %>.addBatch();
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
<%dbLog.data().addingToBatch(dbLog.var("nb_line"), dbLog.str(dataAction));%>
|
||||
batchSizeCounter_<%=cid%>++;
|
||||
<%
|
||||
}else {
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
deletedCount_<%=cid%> = deletedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().deleting(dbLog.var("nb_line"));%>
|
||||
|
||||
@@ -58,7 +58,7 @@ USER.NAME=Username
|
||||
USE_COMMIT_CONTROL.NAME=Use commit control
|
||||
USE_EXISTING_CONNECTION.NAME=Using existing connection
|
||||
USE_FIELD_OPTIONS.NAME=Use field options
|
||||
ENABLE_DEBUG_MODE.NAME=Enable debug mode
|
||||
ENABLE_DEBUG_MODE.NAME=Debug query mode
|
||||
DB_VERSION.NAME=DB Version
|
||||
DB_VERSION.ITEM.V5R2_V5R4=V5R2 to V5R4 (Deprecated)
|
||||
DB_VERSION.ITEM.V5R3_V6R1=V5R3 to V6R1 (Deprecated)
|
||||
|
||||
@@ -123,6 +123,9 @@
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED="true"/>
|
||||
<IMPORT NAME="common-logging1.1.3" MODULE="commons-logging-1.1.3.jar" MVN="mvn:commons-logging/commons-logging/1.1.3" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.3.jar" REQUIRED="true"/>
|
||||
|
||||
<IMPORT NAME="slf4j-api-1.7.25.jar" MODULE="slf4j-api-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-api/1.7.25" REQUIRED_IF="USE_SHARED_CONNECTION =='true'" />
|
||||
<IMPORT NAME="slf4j-log4j12-1.7.25.jar" MODULE="slf4j-log4j12-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-log4j12/1.7.25" REQUIRED_IF="USE_SHARED_CONNECTION =='true'" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -139,6 +139,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
@@ -228,6 +229,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
@@ -345,6 +347,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
@@ -406,6 +409,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
%>
|
||||
try{
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
@@ -501,6 +505,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updateFlag_<%=cid%>=pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
if(updateFlag_<%=cid%> != 0) {
|
||||
nb_line_<%=cid%>++;
|
||||
@@ -567,6 +572,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
@@ -645,6 +651,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
deletedCount_<%=cid%> = deletedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().deleting(dbLog.var("nb_line"));%>
|
||||
|
||||
@@ -52,7 +52,7 @@ FIELD_OPTIONS.ITEM.UPDATE_KEY=Key in update
|
||||
FIELD_OPTIONS.ITEM.DELETE_KEY=Key in delete
|
||||
FIELD_OPTIONS.ITEM.UPDATABLE=Updatable
|
||||
FIELD_OPTIONS.ITEM.INSERTABLE=Insertable
|
||||
ENABLE_DEBUG_MODE.NAME=Enable debug mode
|
||||
ENABLE_DEBUG_MODE.NAME=Debug query mode
|
||||
|
||||
NB_LINE.NAME=Number of line
|
||||
NB_LINE_UPDATED.NAME= NB Line Updated
|
||||
|
||||
@@ -215,6 +215,8 @@
|
||||
</TEMPLATES>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-MysqlConnector" MODULE="mysql-connector-java-5.1.30-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.30-bin/6.0.0" />
|
||||
<IMPORT NAME="slf4j-api-1.7.25.jar" MODULE="slf4j-api-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-api/1.7.25" REQUIRED_IF="(USE_SHARED_CONNECTION == 'true' AND SPECIFY_DATASOURCE_ALIAS=='false')" />
|
||||
<IMPORT NAME="slf4j-log4j12-1.7.25.jar" MODULE="slf4j-log4j12-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-log4j12/1.7.25" REQUIRED_IF="(USE_SHARED_CONNECTION == 'true' AND SPECIFY_DATASOURCE_ALIAS=='false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS/>
|
||||
|
||||
@@ -73,7 +73,7 @@ FIELD_OPTIONS.ITEM.DELETE_KEY=Key in delete
|
||||
FIELD_OPTIONS.ITEM.UPDATABLE=Updatable
|
||||
FIELD_OPTIONS.ITEM.INSERTABLE=Insertable
|
||||
PROPERTIES.NAME=Additional JDBC Parameters
|
||||
ENABLE_DEBUG_MODE.NAME=Enable debug mode
|
||||
ENABLE_DEBUG_MODE.NAME=Debug query mode
|
||||
|
||||
NB_LINE_UPDATED.NAME=Number Of Updated Lines
|
||||
NB_LINE_INSERTED.NAME=Number Of Inserted Lines
|
||||
|
||||
@@ -121,14 +121,14 @@
|
||||
<IMPORT NAME="aws-java-sdk-1.11.406.jar" MODULE="aws-java-sdk-1.11.406.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.406"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.9.9.jar" MODULE="jackson-core-2.9.9.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.9"
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind-2.9.9.jar" MODULE="jackson-databind-2.9.9.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.9.9"
|
||||
<IMPORT NAME="jackson-databind-2.10.1.jar" MODULE="jackson-databind-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations-2.9.0.jar" MODULE="jackson-annotations-2.9.0.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.9.0"
|
||||
<IMPORT NAME="jackson-annotations-2.10.1.jar" MODULE="jackson-annotations-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpcore-4.4.9.jar" MODULE="httpcore-4.4.9.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.9"
|
||||
|
||||
@@ -427,14 +427,14 @@
|
||||
<IMPORT NAME="aws-java-sdk-1.11.406.jar" MODULE="aws-java-sdk-1.11.406.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.406"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.9.9.jar" MODULE="jackson-core-2.9.9.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.9"
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind-2.9.9.jar" MODULE="jackson-databind-2.9.9.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.9.9"
|
||||
<IMPORT NAME="jackson-databind-2.10.1.jar" MODULE="jackson-databind-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations-2.9.0.jar" MODULE="jackson-annotations-2.9.0.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.9.0"
|
||||
<IMPORT NAME="jackson-annotations-2.10.1.jar" MODULE="jackson-annotations-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpcore-4.4.9.jar" MODULE="httpcore-4.4.9.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.9"
|
||||
|
||||
@@ -178,14 +178,14 @@
|
||||
<IMPORT NAME="aws-java-sdk-1.11.406.jar" MODULE="aws-java-sdk-1.11.406.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.406"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.9.9.jar" MODULE="jackson-core-2.9.9.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.9"
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind-2.9.9.jar" MODULE="jackson-databind-2.9.9.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.9.9"
|
||||
<IMPORT NAME="jackson-databind-2.10.1.jar" MODULE="jackson-databind-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations-2.9.0.jar" MODULE="jackson-annotations-2.9.0.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.9.0"
|
||||
<IMPORT NAME="jackson-annotations-2.10.1.jar" MODULE="jackson-annotations-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpcore-4.4.9.jar" MODULE="httpcore-4.4.9.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.9"
|
||||
|
||||
@@ -164,6 +164,8 @@
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-MysqlConnector" MODULE="mysql-connector-java-5.1.30-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.30-bin/6.0.0" REQUIRED_IF="DB_VERSION == 'MYSQL_5'" />
|
||||
<IMPORT NAME="Driver-MysqlConnector_Old" MODULE="mysql-connector-java-5.1.0-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.0-bin/6.0.0" REQUIRED="false" />
|
||||
<IMPORT NAME="slf4j-api-1.7.25.jar" MODULE="slf4j-api-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-api/1.7.25" REQUIRED_IF="USE_SHARED_CONNECTION =='true'" />
|
||||
<IMPORT NAME="slf4j-log4j12-1.7.25.jar" MODULE="slf4j-log4j12-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-log4j12/1.7.25" REQUIRED_IF="USE_SHARED_CONNECTION =='true'" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS/>
|
||||
|
||||
@@ -763,6 +763,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
pstmt_<%=cid %>.addBatch();
|
||||
nb_line_<%=cid%>++;
|
||||
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
<%dbLog.data().addingToBatch(dbLog.var("nb_line"), dbLog.str(dataAction));%>
|
||||
<%
|
||||
if (!("").equals(numPerInsert ) && !("0").equals(numPerInsert )) {
|
||||
@@ -776,6 +777,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
%>
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -853,6 +855,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
%>
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -979,6 +982,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
%>
|
||||
pstmt_<%=cid %>.addBatch();
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
<%dbLog.data().addingToBatch(dbLog.var("nb_line"), dbLog.str(dataAction));%>
|
||||
batchSizeCounter_<%=cid%>++;
|
||||
<%
|
||||
@@ -986,6 +990,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
%>
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -1137,6 +1142,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
}
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
@@ -1212,6 +1218,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
}
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
@@ -1389,11 +1396,13 @@ if(columnList != null && columnList.size() > 0) {
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
}else{
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
}
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -1472,12 +1481,14 @@ if(columnList != null && columnList.size() > 0) {
|
||||
%>
|
||||
pstmt_<%=cid %>.addBatch();
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
<%dbLog.data().addingToBatch(dbLog.var("nb_line"), dbLog.str(dataAction));%>
|
||||
batchSizeCounter_<%=cid%>++;
|
||||
<%
|
||||
}else {
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
deletedCount_<%=cid%> = deletedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().deleting(dbLog.var("nb_line"));%>
|
||||
@@ -1625,6 +1636,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
%>
|
||||
int count_on_duplicate_key_<%=cid%> = 0;
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
count_on_duplicate_key_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().insertingOnDuplicateKeyUpdating(null);%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -1690,6 +1702,7 @@ if(columnList != null && columnList.size() > 0) {
|
||||
%>
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().insertingOnDuplicateKeyUpdating(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
|
||||
@@ -82,7 +82,7 @@ FIELD_OPTIONS.ITEM.DELETE_KEY=Key in delete
|
||||
FIELD_OPTIONS.ITEM.UPDATABLE=Updatable
|
||||
FIELD_OPTIONS.ITEM.INSERTABLE=Insertable
|
||||
PROPERTIES.NAME=Additional JDBC Parameters
|
||||
ENABLE_DEBUG_MODE.NAME=Enable debug mode
|
||||
ENABLE_DEBUG_MODE.NAME=Debug query mode
|
||||
|
||||
NB_LINE_UPDATED.NAME=Number Of Updated Lines
|
||||
NB_LINE_INSERTED.NAME=Number Of Inserted Lines
|
||||
|
||||
@@ -56,7 +56,7 @@ imports="
|
||||
if(isLog4jEnabled){
|
||||
shareDBClass = "SharedDBConnectionLog4j";
|
||||
%>
|
||||
SharedDBConnectionLog4j.initLogger(log,"<%=cid%>");
|
||||
SharedDBConnectionLog4j.initLogger(log.getName(),"<%=cid%>");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -243,6 +243,8 @@
|
||||
<IMPORT NAME="Driver-Ojdbc8" MODULE="ojdbc8-12.2.0.1.jar" MVN="mvn:com.oracle.jdbc/ojdbc8/12.2.0.1" REQUIRED_IF="(DB_VERSION == 'ORACLE_18')" />
|
||||
<IMPORT NAME="Driver-Ojdbc7" MODULE="ojdbc7.jar" MVN="mvn:org.talend.libraries/ojdbc7/6.0.0" REQUIRED_IF="(DB_VERSION == 'ORACLE_12')" />
|
||||
<IMPORT NAME="Driver-Ojdbc6" MODULE="ojdbc6.jar" MVN="mvn:org.talend.libraries/ojdbc6/6.0.0" REQUIRED_IF="(DB_VERSION == 'ORACLE_11')" />
|
||||
<IMPORT NAME="slf4j-api-1.7.25.jar" MODULE="slf4j-api-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-api/1.7.25" REQUIRED_IF="USE_SHARED_CONNECTION =='true'" />
|
||||
<IMPORT NAME="slf4j-log4j12-1.7.25.jar" MODULE="slf4j-log4j12-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-log4j12/1.7.25" REQUIRED_IF="USE_SHARED_CONNECTION =='true'" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -453,6 +453,7 @@ IMetadataColumn talendDynCol = null;
|
||||
%>
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -594,6 +595,7 @@ IMetadataColumn talendDynCol = null;
|
||||
%>
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -758,6 +760,7 @@ IMetadataColumn talendDynCol = null;
|
||||
}
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
@@ -838,6 +841,7 @@ IMetadataColumn talendDynCol = null;
|
||||
}
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
@@ -1023,11 +1027,13 @@ IMetadataColumn talendDynCol = null;
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
}else{
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
}
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -1116,6 +1122,7 @@ IMetadataColumn talendDynCol = null;
|
||||
}else {
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
deletedCount_<%=cid%> = deletedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().deleting(dbLog.var("nb_line"));%>
|
||||
|
||||
@@ -77,7 +77,7 @@ FIELD_OPTIONS.ITEM.UPDATABLE=Updatable
|
||||
FIELD_OPTIONS.ITEM.INSERTABLE=Insertable
|
||||
LOCAL_SERVICE_NAME.NAME=Local Service Name
|
||||
CONVERT_COLUMN_TABLE_TO_UPPERCASE.NAME=Convert columns and table to uppercase
|
||||
ENABLE_DEBUG_MODE.NAME=Enable debug mode
|
||||
ENABLE_DEBUG_MODE.NAME=Debug query mode
|
||||
DB_VERSION.NAME=DB Version
|
||||
DB_VERSION.ITEM.ORACLE_18=Oracle 18
|
||||
DB_VERSION.ITEM.ORACLE_12=Oracle 12-7
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
|
||||
<FAMILIES>
|
||||
<FAMILY>Databases/DB Specifics/Amazon/Redshift</FAMILY>
|
||||
<FAMILY>Cloud/Amazon/Redshift</FAMILY>
|
||||
</FAMILIES>
|
||||
|
||||
<DOCUMENTATION>
|
||||
@@ -233,14 +234,14 @@
|
||||
<IMPORT NAME="aws-java-sdk-1.11.406.jar" MODULE="aws-java-sdk-1.11.406.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.406"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.9.9.jar" MODULE="jackson-core-2.9.9.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.9"
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind-2.9.9.jar" MODULE="jackson-databind-2.9.9.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.9.9"
|
||||
<IMPORT NAME="jackson-databind-2.10.1.jar" MODULE="jackson-databind-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations-2.9.0.jar" MODULE="jackson-annotations-2.9.0.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.9.0"
|
||||
<IMPORT NAME="jackson-annotations-2.10.1.jar" MODULE="jackson-annotations-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpcore-4.4.9.jar" MODULE="httpcore-4.4.9.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.9"
|
||||
|
||||
@@ -24,13 +24,15 @@
|
||||
String dataset = ElementParameterParser.getValue(node, "__DATASET__");
|
||||
String table = ElementParameterParser.getValue(node, "__TABLE__");
|
||||
String gsFile = ElementParameterParser.getValue(node, "__GS_FILE__");
|
||||
boolean setFieldDelimiter = ElementParameterParser.getBooleanValue(node, "__SET_FIELD_DELIMITER__");
|
||||
String fieldDelimiter = ElementParameterParser.getValue(node, "__FIELD_DELIMITER__");
|
||||
String actionOnData = ElementParameterParser.getValue(node, "__ACTION_ON_DATA__");
|
||||
boolean dieOnError = "true".equals(ElementParameterParser.getValue(node, "__DIE_ON_ERROR__"));
|
||||
|
||||
boolean dieOnError = ElementParameterParser.getBooleanValue(node, "__DIE_ON_ERROR__");
|
||||
boolean createTableIfNotExist = ElementParameterParser.getBooleanValue(node, "__CREATE_TABLE_IF_NOT_EXIST__");
|
||||
String gsFileHeader = ElementParameterParser.getValue(node, "__GS_FILE_HEADER__");
|
||||
String tokenFile = ElementParameterParser.getValue(node,"__TOKEN_NAME__");
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
boolean isLog4jEnabled = ElementParameterParser.getBooleanValue(node.getProcess(), "__LOG4J_ACTIVATE__");
|
||||
boolean dropTable = ElementParameterParser.getBooleanValue(node, "__DROP__");
|
||||
|
||||
String passwordFieldName = "";
|
||||
|
||||
@@ -201,7 +203,7 @@
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
}
|
||||
boolean bulkFileAlreadyExists = "true".equals(ElementParameterParser.getValue(node, "__BULK_FILE_ALREADY_EXIST__"));
|
||||
boolean bulkFileAlreadyExists = ElementParameterParser.getBooleanValue(node, "__BULK_FILE_ALREADY_EXIST__");
|
||||
String accessKey = ElementParameterParser.getValue(node, "__GS_ACCESS_KEY__");
|
||||
String secretKey = ElementParameterParser.getValue(node, "__GS_SECRET_KEY__");
|
||||
String localFilename = ElementParameterParser.getValue(node, "__GS_LOCAL_FILE__");
|
||||
@@ -234,6 +236,7 @@
|
||||
}
|
||||
%>
|
||||
gsService_<%=cid%>.putObject(<%=bucketName%>, fileObject_<%=cid%>);
|
||||
/* ----END-UPLOADING-FILE---- */
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
@@ -242,7 +245,6 @@
|
||||
}
|
||||
}
|
||||
%>
|
||||
/* ----END-UPLOADING-FILE---- */
|
||||
|
||||
<%
|
||||
if (authMode.equals("OAUTH")) {
|
||||
@@ -259,98 +261,130 @@
|
||||
com.google.api.services.bigquery.model.JobConfiguration config_<%=cid%> = new com.google.api.services.bigquery.model.JobConfiguration();
|
||||
com.google.api.services.bigquery.model.JobConfigurationLoad queryLoad_<%=cid%> = new com.google.api.services.bigquery.model.JobConfigurationLoad();
|
||||
|
||||
if (<%=ElementParameterParser.getBooleanValue(node, "__CREATE_TABLE_IF_NOT_EXIST__")%>) {
|
||||
com.google.api.services.bigquery.model.TableSchema schema_<%=cid%> = new com.google.api.services.bigquery.model.TableSchema();
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Table field schema:");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
java.util.List<com.google.api.services.bigquery.model.TableFieldSchema> fields_<%=cid%> = new java.util.ArrayList<com.google.api.services.bigquery.model.TableFieldSchema>();
|
||||
<%
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
if ((metadatas!=null) && (metadatas.size() > 0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
if (metadata != null) {
|
||||
List<IMetadataColumn> columns = metadata.getListColumns();
|
||||
int nbColumns = columns.size();
|
||||
for (int i = 0; i < nbColumns; i++ ) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
String columnName = column.getLabel();
|
||||
String typeToGenerate = "string";
|
||||
if("id_Float".equals(column.getTalendType()) || "id_Double".equals(column.getTalendType())) {
|
||||
typeToGenerate = "float";
|
||||
}else if("id_Integer".equals(column.getTalendType()) || "id_Long".equals(column.getTalendType()) || "id_Short".equals(column.getTalendType())) {
|
||||
typeToGenerate = "integer";
|
||||
} else if("id_Character".equals(column.getTalendType())) {
|
||||
typeToGenerate = "string";
|
||||
} else if("id_BigDecimal".equals(column.getTalendType())) {
|
||||
typeToGenerate = "numeric";
|
||||
} else if("id_Boolean".equals(column.getTalendType())) {
|
||||
typeToGenerate = "boolean";
|
||||
} else if("id_Date".equals(column.getTalendType())) {
|
||||
String pattern = column.getPattern();
|
||||
if(pattern.length() == 12 || pattern.isEmpty() || "\"\"".equals(pattern)) {
|
||||
typeToGenerate = "date";
|
||||
}else if(pattern.length() > 12){
|
||||
typeToGenerate = "timestamp";
|
||||
}else{
|
||||
typeToGenerate = "string";
|
||||
}
|
||||
}
|
||||
%>
|
||||
<%
|
||||
String modeType = null;
|
||||
if (!column.isNullable()) {
|
||||
modeType = "REQUIRED";
|
||||
} else {
|
||||
modeType = "NULLABLE";
|
||||
}
|
||||
%>
|
||||
com.google.api.services.bigquery.model.TableFieldSchema <%=columnName%>_<%=cid%> = new com.google.api.services.bigquery.model.TableFieldSchema();
|
||||
<%=columnName%>_<%=cid%>.setName("<%=columnName%>");
|
||||
<%=columnName%>_<%=cid%>.setType("<%=typeToGenerate%>");
|
||||
<%=columnName%>_<%=cid%>.setMode("<%=modeType%>");
|
||||
fields_<%=cid%>.add(<%=columnName%>_<%=cid%>);
|
||||
<%
|
||||
<%if (dropTable) {%>
|
||||
try {
|
||||
bigqueryclient_<%=cid%>.tables().delete(PROJECT_ID_<%=cid%>, <%=dataset%>, <%=table%>).execute();
|
||||
} catch (com.google.api.client.googleapis.json.GoogleJsonResponseException e_<%=cid%>) {
|
||||
if (e_<%=cid%>.getDetails().getCode() != 404) {
|
||||
<%
|
||||
if(dieOnError) {
|
||||
%>
|
||||
throw e_<%=cid%>;
|
||||
<%
|
||||
} else {
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.debug("<%=cid%> - Field index[<%=i%>] {\"name\":\"<%=columnName%>\",\"type\":\"<%=typeToGenerate%>\",\"mode\":\"<%=modeType%>\"}");
|
||||
log.error("<%=cid%> - "+ e_<%=cid%>.getDetails().getMessage());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
System.err.println(e_<%=cid%>.getDetails().getMessage());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
<%}%>
|
||||
|
||||
schema_<%=cid%>.setFields(fields_<%=cid%>);
|
||||
|
||||
queryLoad_<%=cid%>.setSchema(schema_<%=cid%>);
|
||||
<%if (createTableIfNotExist) { %>
|
||||
|
||||
}
|
||||
<%
|
||||
if("true".equals(ElementParameterParser.getValue(node, "__CREATE_TABLE_IF_NOT_EXIST__"))) {
|
||||
%>
|
||||
queryLoad_<%=cid%>.setCreateDisposition("CREATE_IF_NEEDED");
|
||||
try {
|
||||
com.google.api.services.bigquery.model.Table getTable = bigqueryclient_<%=cid%>.tables().get(PROJECT_ID_<%=cid%>, <%=dataset%>, <%=table%>).execute();
|
||||
queryLoad_<%=cid%>.setCreateDisposition("CREATE_NEVER");
|
||||
} catch (com.google.api.client.googleapis.json.GoogleJsonResponseException e_<%=cid%>){
|
||||
if (e_<%=cid%>.getDetails().getCode() == 404) {
|
||||
com.google.api.services.bigquery.model.TableSchema schema_<%=cid%> = new com.google.api.services.bigquery.model.TableSchema();
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Table field schema:");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
java.util.List<com.google.api.services.bigquery.model.TableFieldSchema> fields_<%=cid%> = new java.util.ArrayList<com.google.api.services.bigquery.model.TableFieldSchema>();
|
||||
<%
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
if ((metadatas!=null) && (metadatas.size() > 0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
if (metadata != null) {
|
||||
List<IMetadataColumn> columns = metadata.getListColumns();
|
||||
int nbColumns = columns.size();
|
||||
for (int i = 0; i < nbColumns; i++ ) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
String columnName = column.getLabel();
|
||||
String typeToGenerate = "string";
|
||||
if("id_Float".equals(column.getTalendType()) || "id_Double".equals(column.getTalendType())) {
|
||||
typeToGenerate = "float";
|
||||
}else if("id_Integer".equals(column.getTalendType()) || "id_Long".equals(column.getTalendType()) || "id_Short".equals(column.getTalendType())) {
|
||||
typeToGenerate = "integer";
|
||||
} else if("id_Character".equals(column.getTalendType())) {
|
||||
typeToGenerate = "string";
|
||||
} else if("id_BigDecimal".equals(column.getTalendType())) {
|
||||
typeToGenerate = "numeric";
|
||||
} else if("id_Boolean".equals(column.getTalendType())) {
|
||||
typeToGenerate = "boolean";
|
||||
} else if("id_Date".equals(column.getTalendType())) {
|
||||
String pattern = column.getPattern();
|
||||
if(pattern.length() == 12 || pattern.isEmpty() || "\"\"".equals(pattern)) {
|
||||
typeToGenerate = "date";
|
||||
}else if(pattern.length() > 12){
|
||||
typeToGenerate = "timestamp";
|
||||
}else{
|
||||
typeToGenerate = "string";
|
||||
}
|
||||
}
|
||||
String modeType = (!column.isNullable()) ? "REQUIRED" : "NULLABLE";
|
||||
%>
|
||||
com.google.api.services.bigquery.model.TableFieldSchema <%=columnName%>_<%=cid%> = new com.google.api.services.bigquery.model.TableFieldSchema();
|
||||
<%=columnName%>_<%=cid%>.setName("<%=columnName%>");
|
||||
<%=columnName%>_<%=cid%>.setType("<%=typeToGenerate%>");
|
||||
<%=columnName%>_<%=cid%>.setMode("<%=modeType%>");
|
||||
fields_<%=cid%>.add(<%=columnName%>_<%=cid%>);
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.debug("<%=cid%> - Field index[<%=i%>] {\"name\":\"<%=columnName%>\",\"type\":\"<%=typeToGenerate%>\",\"mode\":\"<%=modeType%>\"}");
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
schema_<%=cid%>.setFields(fields_<%=cid%>);
|
||||
queryLoad_<%=cid%>.setSchema(schema_<%=cid%>);
|
||||
queryLoad_<%=cid%>.setCreateDisposition("CREATE_IF_NEEDED");
|
||||
} else {
|
||||
<%
|
||||
if(dieOnError) {
|
||||
%>
|
||||
throw e_<%=cid%>;
|
||||
<%
|
||||
} else {
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.error("<%=cid%> - "+ e_<%=cid%>.getDetails().getMessage());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
System.err.println(e_<%=cid%>.getDetails().getMessage());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
queryLoad_<%=cid%>.setCreateDisposition("CREATE_NEVER");
|
||||
<%
|
||||
}
|
||||
|
||||
if("true".equals(ElementParameterParser.getValue(node, "__SET_FIELD_DELIMITER__"))) {
|
||||
if(setFieldDelimiter) {
|
||||
%>
|
||||
queryLoad_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
queryLoad_<%=cid%>.setAllowQuotedNewlines(true);
|
||||
|
||||
queryLoad_<%=cid%>.setWriteDisposition("WRITE_<%=actionOnData%>");
|
||||
com.google.api.services.bigquery.model.TableReference destinationTable_<%=cid%> = new com.google.api.services.bigquery.model.TableReference();
|
||||
destinationTable_<%=cid%>.setProjectId(PROJECT_ID_<%=cid%>);
|
||||
@@ -358,8 +392,8 @@
|
||||
destinationTable_<%=cid%>.setTableId(<%=table%>);
|
||||
|
||||
queryLoad_<%=cid%>.setDestinationTable(destinationTable_<%=cid%>);
|
||||
queryLoad_<%=cid%>.setSourceUris(java.util.Arrays.asList(<%=ElementParameterParser.getValue(node, "__GS_FILE__")%>));
|
||||
queryLoad_<%=cid%>.setSkipLeadingRows(<%=ElementParameterParser.getValue(node, "__GS_FILE_HEADER__")%>);
|
||||
queryLoad_<%=cid%>.setSourceUris(java.util.Arrays.asList(<%=gsFile%>));
|
||||
queryLoad_<%=cid%>.setSkipLeadingRows(<%=gsFileHeader%>);
|
||||
queryLoad_<%=cid%>.setNullMarker("\\N");
|
||||
config_<%=cid%>.setLoad(queryLoad_<%=cid%>);
|
||||
|
||||
@@ -463,9 +497,11 @@
|
||||
%>
|
||||
/* ----START-CREATING-JOB (Cloud API)---- */
|
||||
com.google.cloud.bigquery.TableId tableId_<%=cid%> = com.google.cloud.bigquery.TableId.of(<%=projectId%>, <%=dataset%>, <%=table%>);
|
||||
com.google.cloud.bigquery.Table table_<%=cid%> = bigquery_<%=cid%>.getTable(tableId_<%=cid%>);
|
||||
com.google.cloud.bigquery.LoadJobConfiguration.Builder loadJobBuilder_<%=cid%> = com.google.cloud.bigquery.LoadJobConfiguration.newBuilder(tableId_<%=cid%>, <%=gsFile%>);
|
||||
if (<%=ElementParameterParser.getBooleanValue(node, "__DROP__")%> && table_<%=cid%> != null) {
|
||||
|
||||
boolean dropTable_<%=cid%> = <%=dropTable%>;
|
||||
|
||||
if (dropTable_<%=cid%> && bigquery_<%=cid%>.getTable(tableId_<%=cid%>) != null) {
|
||||
boolean deleted = bigquery_<%=cid%>.delete(tableId_<%=cid%>);
|
||||
if (deleted) {
|
||||
<%
|
||||
@@ -479,123 +515,100 @@
|
||||
throw new RuntimeException("Unable to delete table " + tableId_<%=cid%>);
|
||||
}
|
||||
}
|
||||
if (<%=ElementParameterParser.getBooleanValue(node, "__DROP__")%> || <%=ElementParameterParser.getBooleanValue(node, "__CREATE_TABLE_IF_NOT_EXIST__")%>) {
|
||||
java.util.List<com.google.cloud.bigquery.Field> fields_<%=cid%> = new java.util.ArrayList<>();
|
||||
<%
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
if ((metadatas!=null) && (metadatas.size() > 0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
if (metadata != null) {
|
||||
List<IMetadataColumn> columns = metadata.getListColumns();
|
||||
int nbColumns = columns.size();
|
||||
for (int i = 0; i < nbColumns; i++ ) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
String columnName = column.getLabel();
|
||||
String typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
|
||||
if("id_String".equals(column.getTalendType()) || "id_Character".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
|
||||
} else if ("id_Float".equals(column.getTalendType()) || "id_Double".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.FLOAT";
|
||||
} else if ("id_Short".equals(column.getTalendType()) || "id_Integer".equals(column.getTalendType()) || "id_Long".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.INTEGER";
|
||||
} else if ("id_BigDecimal".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.NUMERIC";
|
||||
} else if ("id_Boolean".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.BOOLEAN";
|
||||
} else if ("id_Date".equals(column.getTalendType())) {
|
||||
String pattern = column.getPattern();
|
||||
if(pattern.length() == 12 || pattern.isEmpty() || "\"\"".equals(pattern)) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.DATE";
|
||||
}else if(pattern.length() > 12){
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.TIMESTAMP";
|
||||
}else{
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
|
||||
}
|
||||
}
|
||||
|
||||
String modeType = "NULLABLE";
|
||||
if (!column.isNullable()) {
|
||||
modeType = "REQUIRED";
|
||||
}
|
||||
|
||||
<%if(createTableIfNotExist){%>
|
||||
if(bigquery_<%=cid%>.getTable(tableId_<%=cid%>) == null){
|
||||
java.util.List<com.google.cloud.bigquery.Field> fields_<%=cid%> = new java.util.ArrayList<>();
|
||||
<%
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
if ((metadatas!=null) && (metadatas.size() > 0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
if (metadata != null) {
|
||||
List<IMetadataColumn> columns = metadata.getListColumns();
|
||||
int nbColumns = columns.size();
|
||||
for (int i = 0; i < nbColumns; i++ ) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
String columnName = column.getLabel();
|
||||
String typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
|
||||
if("id_String".equals(column.getTalendType()) || "id_Character".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
|
||||
} else if ("id_Float".equals(column.getTalendType()) || "id_Double".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.FLOAT";
|
||||
} else if ("id_Short".equals(column.getTalendType()) || "id_Integer".equals(column.getTalendType()) || "id_Long".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.INTEGER";
|
||||
} else if ("id_BigDecimal".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.NUMERIC";
|
||||
} else if ("id_Boolean".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.BOOLEAN";
|
||||
} else if ("id_Date".equals(column.getTalendType())) {
|
||||
String pattern = column.getPattern();
|
||||
if(pattern.length() == 12 || pattern.isEmpty() || "\"\"".equals(pattern)) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.DATE";
|
||||
}else if(pattern.length() > 12){
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.TIMESTAMP";
|
||||
}else{
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.STRING";
|
||||
}
|
||||
}
|
||||
String modeType = (!column.isNullable()) ? "REQUIRED" : "NULLABLE";
|
||||
%>
|
||||
|
||||
com.google.cloud.bigquery.Field field_<%=i%> = com.google.cloud.bigquery.Field.newBuilder("<%=columnName%>", <%=typeToGenerate%>)
|
||||
.setMode(com.google.cloud.bigquery.Field.Mode.valueOf("<%=modeType%>"))
|
||||
.build();
|
||||
fields_<%=cid%>.add(field_<%=i%>);
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.debug("<%=cid%> - Field index[<%=i%>] {\"name\":\"<%=columnName%>\",\"type\":\"<%=typeToGenerate%>\"}");
|
||||
<%
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.debug("<%=cid%> - Field index[<%=i%>] {\"name\":\"<%=columnName%>\",\"type\":\"<%=typeToGenerate%>\"}");
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
%>
|
||||
|
||||
com.google.cloud.bigquery.Schema schema_<%=cid%> = com.google.cloud.bigquery.Schema.of(fields_<%=cid%>);
|
||||
com.google.cloud.bigquery.TableInfo tableInfo_<%=cid%> = com.google.cloud.bigquery.TableInfo.newBuilder(tableId_<%=cid%>, com.google.cloud.bigquery.StandardTableDefinition.of(schema_<%=cid%>)).build();
|
||||
table_<%=cid%> = bigquery_<%=cid%>.create(tableInfo_<%=cid%>);
|
||||
loadJobBuilder_<%=cid%>.setSchema(schema_<%=cid%>);
|
||||
}
|
||||
|
||||
<%
|
||||
if("true".equals(ElementParameterParser.getValue(node, "__CREATE_TABLE_IF_NOT_EXIST__"))) {
|
||||
%>
|
||||
com.google.cloud.bigquery.Schema schema_<%=cid%> = com.google.cloud.bigquery.Schema.of(fields_<%=cid%>);
|
||||
com.google.cloud.bigquery.TableInfo tableInfo_<%=cid%> = com.google.cloud.bigquery.TableInfo.newBuilder(tableId_<%=cid%>, com.google.cloud.bigquery.StandardTableDefinition.of(schema_<%=cid%>)).build();
|
||||
com.google.cloud.bigquery.Table table_<%=cid%> = bigquery_<%=cid%>.create(tableInfo_<%=cid%>);
|
||||
loadJobBuilder_<%=cid%>.setSchema(schema_<%=cid%>);
|
||||
loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_IF_NEEDED);
|
||||
<%
|
||||
|
||||
} else {
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_NEVER);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%} else {%>
|
||||
loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_NEVER);
|
||||
<%}%>
|
||||
|
||||
<%
|
||||
if("APPEND".equals(actionOnData)) {
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_APPEND);
|
||||
<%
|
||||
} else if("TRUNCATE".equals(actionOnData)) {
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_TRUNCATE);
|
||||
|
||||
<%
|
||||
}
|
||||
else {
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_EMPTY);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
loadJobBuilder_<%=cid%>.setDestinationTable(tableId_<%=cid%>);
|
||||
com.google.cloud.bigquery.CsvOptions.Builder csvOptions_<%=cid%> = com.google.cloud.bigquery.CsvOptions.newBuilder();
|
||||
csvOptions_<%=cid%>.setAllowQuotedNewLines(true);
|
||||
loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_<%=actionOnData%>);
|
||||
loadJobBuilder_<%=cid%>.setDestinationTable(tableId_<%=cid%>);
|
||||
com.google.cloud.bigquery.CsvOptions.Builder csvOptions_<%=cid%> = com.google.cloud.bigquery.CsvOptions.newBuilder();
|
||||
csvOptions_<%=cid%>.setAllowQuotedNewLines(true);
|
||||
csvOptions_<%=cid%>.setSkipLeadingRows(<%=gsFileHeader%>);
|
||||
|
||||
<%if("true".equals(ElementParameterParser.getValue(node, "__SET_FIELD_DELIMITER__"))) {
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setFormatOptions(csvOptions_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>).build());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
loadJobBuilder_<%=cid%>.setNullMarker("\\N");
|
||||
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.of(loadJobBuilder_<%=cid%>.build()));
|
||||
job_<%=cid%> = job_<%=cid%>.waitFor(com.google.cloud.RetryOption.initialRetryDelay(org.threeten.bp.Duration.ofSeconds(1)));
|
||||
if (job_<%=cid%> != null && job_<%=cid%>.getStatus().getError() == null) {
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Job execution status: " + job_<%=cid%>.getStatus());
|
||||
<%
|
||||
<%if(setFieldDelimiter) {
|
||||
%>
|
||||
csvOptions_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setFormatOptions(csvOptions_<%=cid%>.build());
|
||||
loadJobBuilder_<%=cid%>.setNullMarker("\\N");
|
||||
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.of(loadJobBuilder_<%=cid%>.build()));
|
||||
job_<%=cid%> = job_<%=cid%>.waitFor(com.google.cloud.RetryOption.initialRetryDelay(org.threeten.bp.Duration.ofSeconds(1)));
|
||||
if (job_<%=cid%> != null && job_<%=cid%>.getStatus().getError() == null) {
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Job execution status: " + job_<%=cid%>.getStatus());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} else {
|
||||
List<com.google.cloud.bigquery.BigQueryError> errorList = job_<%=cid%>.getStatus().getExecutionErrors();
|
||||
throw new RuntimeException("Job failed: " + errorList.get(errorList.size() - 1));
|
||||
}
|
||||
%>
|
||||
} else {
|
||||
List<com.google.cloud.bigquery.BigQueryError> errorList = job_<%=cid%>.getStatus().getExecutionErrors();
|
||||
throw new RuntimeException("Job failed: " + errorList.get(errorList.size() - 1));
|
||||
}
|
||||
|
||||
/* ----END-CREATING-JOB (Cloud API)---- */
|
||||
<%
|
||||
|
||||
@@ -173,7 +173,7 @@
|
||||
SHOW_IF="(SET_FIELD_DELIMITER == 'true')">
|
||||
<DEFAULT>","</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="25" SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'">
|
||||
<PARAMETER NAME="DROP" FIELD="CHECK" REQUIRED="true" NUM_ROW="25">
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="ENCODING" FIELD="ENCODING_TYPE" NUM_ROW="30"
|
||||
@@ -190,7 +190,7 @@
|
||||
<IMPORT NAME="google-oauth-client-1.19.0.jar" MODULE="google-oauth-client-1.19.0.jar" MVN="mvn:org.talend.libraries/google-oauth-client-1.19.0/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-oauth-client-1.19.0.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-jackson2-1.19.0.jar" MODULE="google-http-client-jackson2-1.19.0.jar" MVN="mvn:org.talend.libraries/google-http-client-jackson2-1.19.0/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-http-client-jackson2-1.19.0.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="guava-jdk5-13.0.jar" MODULE="guava-jdk5-13.0.jar" MVN="mvn:org.talend.libraries/guava-jdk5-13.0/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.guava/lib/guava-jdk5-13.0.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.9.9.jar" MODULE="jackson-core-2.9.9.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.9" REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1" REQUIRED="true" />
|
||||
<!-- REQUIRED FOR GOOGLE STORAGE -->
|
||||
<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0" REQUIRED="true" />
|
||||
<IMPORT NAME="commons-logging-1.1.1" MODULE="commons-logging-1.1.1.jar" MVN="mvn:org.talend.libraries/commons-logging-1.1.1/6.0.0" UrlPath="platform:/base/plugins/org.apache.commons.logging_1.1.1.v201101211721.jar" REQUIRED="true" />
|
||||
|
||||
@@ -6,45 +6,55 @@
|
||||
|
||||
class BigQueryUtil_<%=cid%> {
|
||||
String projectId;
|
||||
com.google.api.services.bigquery.Bigquery bigqueryclient = null;
|
||||
|
||||
<%=basePackage%>com.google.api.services.bigquery.Bigquery bigqueryclient = null;
|
||||
String tokenFile;
|
||||
boolean useLargeResult = false;
|
||||
String tempDataset;
|
||||
String tempTable;
|
||||
final boolean isAutoGeneratedTemporaryDataset;
|
||||
|
||||
public BigQueryUtil_<%=cid%>(String projectId, com.google.api.services.bigquery.Bigquery bigqueryclient, String tokenFile) {
|
||||
public BigQueryUtil_<%=cid%>(String projectId, <%=basePackage%>com.google.api.services.bigquery.Bigquery bigqueryclient, String tokenFile) {
|
||||
this (projectId, bigqueryclient, tokenFile, null);
|
||||
}
|
||||
|
||||
public BigQueryUtil_<%=cid%>(String projectId, <%=basePackage%>com.google.api.services.bigquery.Bigquery bigqueryclient, String tokenFile, String tempDataset) {
|
||||
this.projectId = projectId;
|
||||
this.bigqueryclient = bigqueryclient;
|
||||
this.tokenFile = tokenFile;
|
||||
this.tempDataset = tempDataset;
|
||||
this.isAutoGeneratedTemporaryDataset = tempDataset == null;
|
||||
}
|
||||
|
||||
|
||||
private String genTempName(String prefix){
|
||||
return "temp_" + prefix + java.util.UUID.randomUUID().toString().replaceAll("-", "") + "<%=cid%>".toLowerCase().replaceAll("[^a-z0-9]", "0").replaceAll("^[^a-z]", "a") + Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
|
||||
}
|
||||
|
||||
|
||||
public void cleanup() throws Exception{
|
||||
if(useLargeResult){
|
||||
bigqueryclient.tables().delete(projectId, tempDataset, tempTable).execute();
|
||||
bigqueryclient.datasets().delete(projectId, tempDataset).execute();
|
||||
if (isAutoGeneratedTemporaryDataset) {
|
||||
bigqueryclient.datasets().delete(projectId, tempDataset).execute();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String getLocation(com.google.api.services.bigquery.model.JobConfigurationQuery queryConfig) throws Exception {
|
||||
private String getLocation(<%=basePackage%>com.google.api.services.bigquery.model.JobConfigurationQuery queryConfig) throws Exception {
|
||||
String location = null;
|
||||
com.google.api.services.bigquery.model.JobConfiguration config = new com.google.api.services.bigquery.model.JobConfiguration();
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.JobConfiguration config = new <%=basePackage%>com.google.api.services.bigquery.model.JobConfiguration();
|
||||
config.setQuery(queryConfig);
|
||||
config.setDryRun(true);
|
||||
com.google.api.services.bigquery.model.Job job = new com.google.api.services.bigquery.model.Job();
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.Job job = new <%=basePackage%>com.google.api.services.bigquery.model.Job();
|
||||
job.setConfiguration(config);
|
||||
List<com.google.api.services.bigquery.model.TableReference> referencedTables = bigqueryclient.jobs().insert(projectId, job).execute().getStatistics().getQuery().getReferencedTables();
|
||||
List<<%=basePackage%>com.google.api.services.bigquery.model.TableReference> referencedTables = bigqueryclient.jobs().insert(projectId, job).execute().getStatistics().getQuery().getReferencedTables();
|
||||
if(referencedTables != null && !referencedTables.isEmpty()) {
|
||||
location = bigqueryclient.tables().get(projectId, referencedTables.get(0).getDatasetId(), referencedTables.get(0).getTableId()).execute().getLocation();
|
||||
location = bigqueryclient.tables().get(referencedTables.get(0).getProjectId(), referencedTables.get(0).getDatasetId(), referencedTables.get(0).getTableId()).execute().getLocation();
|
||||
}
|
||||
return location;
|
||||
}
|
||||
|
||||
|
||||
private void createDataset(String location) throws Exception {
|
||||
com.google.api.services.bigquery.model.Dataset dataset = new com.google.api.services.bigquery.model.Dataset().setDatasetReference(new com.google.api.services.bigquery.model.DatasetReference().setProjectId(projectId).setDatasetId(tempDataset));
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.Dataset dataset = new <%=basePackage%>com.google.api.services.bigquery.model.Dataset().setDatasetReference(new <%=basePackage%>com.google.api.services.bigquery.model.DatasetReference().setProjectId(projectId).setDatasetId(tempDataset));
|
||||
if(location != null) {
|
||||
dataset.setLocation(location);
|
||||
}
|
||||
@@ -54,39 +64,41 @@ class BigQueryUtil_<%=cid%> {
|
||||
bigqueryclient.datasets().insert(projectId, dataset).execute();
|
||||
}
|
||||
|
||||
public com.google.api.services.bigquery.model.Job executeQuery(String query, boolean useLargeResult) throws Exception{
|
||||
public <%=basePackage%>com.google.api.services.bigquery.model.Job executeQuery(String query, boolean useLargeResult) throws Exception{
|
||||
return executeQuery(query, useLargeResult, true);
|
||||
}
|
||||
|
||||
public com.google.api.services.bigquery.model.Job executeQuery(String query, boolean useLargeResult, boolean useLegacySql) throws Exception{
|
||||
com.google.api.services.bigquery.model.JobConfigurationQuery queryConfig = new com.google.api.services.bigquery.model.JobConfigurationQuery();
|
||||
public <%=basePackage%>com.google.api.services.bigquery.model.Job executeQuery(String query, boolean useLargeResult, boolean useLegacySql) throws Exception{
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.JobConfigurationQuery queryConfig = new <%=basePackage%>com.google.api.services.bigquery.model.JobConfigurationQuery();
|
||||
queryConfig.setQuery(query);
|
||||
queryConfig.setUseLegacySql(useLegacySql);
|
||||
String location = getLocation(queryConfig);
|
||||
if(useLargeResult){
|
||||
this.useLargeResult = true;
|
||||
tempDataset = genTempName("dataset");
|
||||
tempTable = genTempName("table");
|
||||
createDataset(location);
|
||||
if (isAutoGeneratedTemporaryDataset) {
|
||||
tempDataset = genTempName("dataset");
|
||||
createDataset(location);
|
||||
}
|
||||
queryConfig.setAllowLargeResults(true);
|
||||
queryConfig.setDestinationTable(new com.google.api.services.bigquery.model.TableReference()
|
||||
queryConfig.setDestinationTable(new <%=basePackage%>com.google.api.services.bigquery.model.TableReference()
|
||||
.setProjectId(projectId)
|
||||
.setDatasetId(tempDataset)
|
||||
.setTableId(tempTable));
|
||||
}
|
||||
|
||||
com.google.api.services.bigquery.model.JobConfiguration config = new com.google.api.services.bigquery.model.JobConfiguration();
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.JobConfiguration config = new <%=basePackage%>com.google.api.services.bigquery.model.JobConfiguration();
|
||||
config.setQuery(queryConfig);
|
||||
|
||||
com.google.api.services.bigquery.model.Job job = new com.google.api.services.bigquery.model.Job();
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.Job job = new <%=basePackage%>com.google.api.services.bigquery.model.Job();
|
||||
job.setConfiguration(config);
|
||||
|
||||
com.google.api.services.bigquery.model.Job insert = null;
|
||||
com.google.api.services.bigquery.model.JobReference jobId = null;
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.Job insert = null;
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.JobReference jobId = null;
|
||||
try {
|
||||
insert = bigqueryclient.jobs().insert(projectId, job).execute();
|
||||
jobId = insert.getJobReference();
|
||||
} catch (com.google.api.client.googleapis.json.GoogleJsonResponseException e) {
|
||||
} catch (<%=basePackage%>com.google.api.client.googleapis.json.GoogleJsonResponseException e) {
|
||||
if(tokenFile != null){
|
||||
try {
|
||||
java.io.File f = new java.io.File(tokenFile);
|
||||
@@ -122,7 +134,7 @@ class BigQueryUtil_<%=cid%> {
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
@@ -132,10 +144,22 @@ class BigQueryUtil_<%=cid%> {
|
||||
%>
|
||||
// wait for query execution
|
||||
while (true) {
|
||||
com.google.api.services.bigquery.model.Job pollJob = bigqueryclient.jobs().get(projectId, jobId.getJobId()).setLocation(location).execute();
|
||||
com.google.api.services.bigquery.model.JobStatus status = pollJob.getStatus();
|
||||
|
||||
<%
|
||||
if("".equals(basePackage)) {
|
||||
%>
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.Job pollJob = bigqueryclient.jobs().get(projectId, jobId.getJobId()).set("location", location).execute();
|
||||
<%
|
||||
}else{
|
||||
%>
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.Job pollJob = bigqueryclient.jobs().get(projectId, jobId.getJobId()).setLocation(location).execute();
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.JobStatus status = pollJob.getStatus();
|
||||
if (status.getState().equals("DONE")) {
|
||||
com.google.api.services.bigquery.model.ErrorProto errorProto = status.getErrorResult();
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.ErrorProto errorProto = status.getErrorResult();
|
||||
if(errorProto != null){// job failed, handle it
|
||||
<%if("AUTO".equals(resultSizeType)){%>
|
||||
if(!useLargeResult && "responseTooLarge".equals(errorProto.getReason())){// try with large result flag
|
||||
@@ -146,7 +170,7 @@ class BigQueryUtil_<%=cid%> {
|
||||
<%
|
||||
}
|
||||
%>
|
||||
return executeQuery(query, true);
|
||||
return executeQuery(query, true, useLegacySql);
|
||||
}
|
||||
<%}%>
|
||||
// Do not throw exception to avoid behavior changed(because it may throw "duplicate" exception which do not throw before);
|
||||
@@ -164,13 +188,12 @@ class BigQueryUtil_<%=cid%> {
|
||||
}// else job successful
|
||||
break;
|
||||
}
|
||||
// Pause execution for one second before polling job status again, to
|
||||
// Pause execution for one second before polling job status again, to
|
||||
// reduce unnecessary calls to the BigQUery API and lower overall
|
||||
// application bandwidth.
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
|
||||
|
||||
return insert;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
<%@ jet
|
||||
imports="
|
||||
org.talend.core.model.process.INode
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
org.talend.core.model.process.INode
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
org.talend.designer.codegen.config.CodeGeneratorArgument
|
||||
org.talend.core.model.metadata.IMetadataTable
|
||||
org.talend.core.model.metadata.IMetadataColumn
|
||||
org.talend.core.model.metadata.IMetadataTable
|
||||
org.talend.core.model.metadata.IMetadataColumn
|
||||
org.talend.core.model.process.IConnection
|
||||
org.talend.core.model.process.IConnectionCategory
|
||||
org.talend.core.model.metadata.types.JavaTypesManager
|
||||
@@ -29,10 +29,13 @@
|
||||
String query = ElementParameterParser.getValue(node,"__QUERY__");
|
||||
boolean useLegacySql = ElementParameterParser.getBooleanValue(node,"__USE_LEGACY_SQL__");
|
||||
String resultSizeType = ElementParameterParser.getValue(node,"__RESULT_SIZE__");
|
||||
String basePackage = "";
|
||||
boolean isCustomTemporaryName = ElementParameterParser.getBooleanValue(node,"__USE_CUSTOM_TEMPORARY_DATASET__");
|
||||
String tempDataset = ElementParameterParser.getValue(node,"__TEMPORARY_DATASET_NAME__");
|
||||
|
||||
query = query.replaceAll("\n"," ");
|
||||
query = query.replaceAll("\r"," ");
|
||||
|
||||
|
||||
String tokenFile = ElementParameterParser.getValue(node,"__TOKEN_NAME__");
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
if (authMode.equals("OAUTH")) {
|
||||
@@ -41,22 +44,22 @@
|
||||
<%
|
||||
String passwordFieldName = "__CLIENT_SECRET__";
|
||||
%>
|
||||
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
|
||||
final String CLIENT_SECRET_<%=cid%> = "{\"web\": {\"client_id\": \""+<%=clientId%>+"\",\"client_secret\": \"" +decryptedPassword_<%=cid%>+ "\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://accounts.google.com/o/oauth2/token\"}}";
|
||||
final String PROJECT_ID_<%=cid %> = <%=projectId %>;
|
||||
|
||||
|
||||
// Static variables for API scope, callback URI, and HTTP/JSON functions
|
||||
final List<String> SCOPES_<%=cid%> = java.util.Arrays.asList("https://www.googleapis.com/auth/bigquery");
|
||||
final String REDIRECT_URI_<%=cid%> = "urn:ietf:wg:oauth:2.0:oob";
|
||||
final com.google.api.client.http.HttpTransport TRANSPORT_<%=cid %> = new com.google.api.client.http.javanet.NetHttpTransport();
|
||||
final com.google.api.client.json.JsonFactory JSON_FACTORY_<%=cid %> = new com.google.api.client.json.jackson2.JacksonFactory();
|
||||
|
||||
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets clientSecrets_<%=cid%> = com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets.load(
|
||||
new com.google.api.client.json.jackson2.JacksonFactory(), new java.io.InputStreamReader(new java.io.ByteArrayInputStream(
|
||||
CLIENT_SECRET_<%=cid%>.getBytes())));
|
||||
|
||||
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow flow_<%=cid%> = null;
|
||||
com.google.api.services.bigquery.Bigquery bigqueryclient_<%=cid%> = null;
|
||||
long nb_line_<%=cid%> = 0;
|
||||
@@ -69,7 +72,7 @@
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
|
||||
// Attempt to load existing refresh token
|
||||
String tokenFile_<%=cid %> = <%=tokenFile%>;
|
||||
java.util.Properties properties_<%=cid%> = new java.util.Properties();
|
||||
@@ -95,16 +98,16 @@
|
||||
%>
|
||||
}
|
||||
String storedRefreshToken_<%=cid%> = (String) properties_<%=cid%>.get("refreshtoken");
|
||||
|
||||
|
||||
// Check to see if the an existing refresh token was loaded.
|
||||
// If so, create a credential and call refreshToken() to get a new
|
||||
// access token.
|
||||
if (storedRefreshToken_<%=cid%> != null) {
|
||||
// Request a new Access token using the refresh token.
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleCredential credential_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2. GoogleCredential.Builder().setTransport(TRANSPORT_<%=cid%>)
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleCredential credential_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder().setTransport(TRANSPORT_<%=cid%>)
|
||||
.setJsonFactory(JSON_FACTORY_<%=cid%>).setClientSecrets(clientSecrets_<%=cid%>)
|
||||
.build().setFromTokenResponse(new com.google.api.client.auth.oauth2.TokenResponse().setRefreshToken(storedRefreshToken_<%=cid%>));
|
||||
|
||||
|
||||
credential_<%=cid%>.refreshToken();
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
@@ -126,7 +129,7 @@
|
||||
if(authorizationCode_<%=cid%> == null || "".equals(authorizationCode_<%=cid%>) || "\"\"".equals(authorizationCode_<%=cid%>)) {
|
||||
String authorizeUrl_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeRequestUrl(
|
||||
clientSecrets_<%=cid%>, REDIRECT_URI_<%=cid%>, SCOPES_<%=cid%>).setState("").build();
|
||||
|
||||
|
||||
System.out
|
||||
.println("Paste this URL into a web browser to authorize BigQuery Access:\n"
|
||||
+ authorizeUrl_<%=cid%>);
|
||||
@@ -156,7 +159,7 @@
|
||||
}
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse response_<%=cid%> = flow_<%=cid%>.newTokenRequest(authorizationCode_<%=cid%>).setRedirectUri(REDIRECT_URI_<%=cid%>).execute();
|
||||
com.google.api.client.auth.oauth2.Credential credential_<%=cid%> = flow_<%=cid%>.createAndStoreCredential(response_<%=cid%>, null);
|
||||
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
@@ -172,15 +175,15 @@
|
||||
if (outputStream_<%=cid%> != null) {
|
||||
outputStream_<%=cid%>.close();
|
||||
}
|
||||
|
||||
|
||||
bigqueryclient_<%=cid%> = new com.google.api.services.bigquery.Bigquery.Builder(new com.google.api.client.http.javanet.NetHttpTransport(),new com.google.api.client.json.jackson2.JacksonFactory(),credential_<%=cid%>).build();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/tBigQueryInput/BigQueryInputQueryHelper.javajet"%>
|
||||
|
||||
|
||||
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/tBigQueryInput/BigQueryInputQueryHelper.javajet"%>
|
||||
|
||||
// Start a Query Job
|
||||
String querySql_<%=cid %> = <%=query %>;
|
||||
System.out.format("Running Query : %s\n", querySql_<%=cid %>);
|
||||
@@ -191,9 +194,9 @@
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
BigQueryUtil_<%=cid%> bigQueryUtil_<%=cid%> = new BigQueryUtil_<%=cid%>(PROJECT_ID_<%=cid%>, bigqueryclient_<%=cid%>, tokenFile_<%=cid%>);
|
||||
|
||||
|
||||
BigQueryUtil_<%=cid%> bigQueryUtil_<%=cid%> = new BigQueryUtil_<%=cid%>(PROJECT_ID_<%=cid%>, bigqueryclient_<%=cid%>, tokenFile_<%=cid%><% if (isCustomTemporaryName) {%>, <%=tempDataset%> <%} %>);
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
@@ -202,7 +205,7 @@
|
||||
}
|
||||
%>
|
||||
com.google.api.services.bigquery.model.Job insert_<%=cid %> = bigQueryUtil_<%=cid%>.executeQuery(querySql_<%=cid%>, <%="LARGE".equals(resultSizeType) ? true : false%>, <%=useLegacySql%>);
|
||||
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
@@ -221,60 +224,99 @@
|
||||
}
|
||||
|
||||
String query_<%=cid%> = <%=query%>;
|
||||
String dataset = query_<%=cid%>.substring(query_<%=cid%>.indexOf("from") + 4, query_<%=cid%>.indexOf(".")).trim();
|
||||
String temp_table = "temp_" + dataset + java.util.UUID.randomUUID().toString().replaceAll("-", "")
|
||||
+ "<%=cid%>".toLowerCase().replaceAll("[^a-z0-9]", "0").replaceAll("^[^a-z]", "a")
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
|
||||
<% if(isLog4jEnabled) { %>
|
||||
log.info("<%=cid%> - query " + <%=query%>);
|
||||
<% } %>
|
||||
|
||||
com.google.cloud.bigquery.BigQuery bigquery_<%=cid%> = com.google.cloud.bigquery.BigQueryOptions.newBuilder()
|
||||
.setCredentials(credentials_<%=cid%>)
|
||||
.setProjectId(<%=projectId%>)
|
||||
.build()
|
||||
.getService();
|
||||
|
||||
com.google.cloud.bigquery.QueryJobConfiguration.Builder queryConfiguration_<%=cid%> = com.google.cloud.bigquery.QueryJobConfiguration.newBuilder(<%=query%>).setUseLegacySql(<%=useLegacySql%>);
|
||||
|
||||
<%
|
||||
if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) {
|
||||
%>
|
||||
queryConfiguration_<%=cid%>.setDestinationTable(com.google.cloud.bigquery.TableId.of(dataset, temp_table)).setAllowLargeResults(true);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId.of(java.util.UUID.randomUUID().toString());
|
||||
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.newBuilder(queryConfiguration_<%=cid%>.build()).setJobId(jobId_<%=cid%>).build());
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
com.google.cloud.bigquery.QueryJobConfiguration.Builder queryConfiguration_<%=cid%> = com.google.cloud.bigquery.QueryJobConfiguration.newBuilder(<%=query%>).setUseLegacySql(<%=useLegacySql%>);
|
||||
|
||||
<%if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) {
|
||||
if (isCustomTemporaryName) {%>
|
||||
String tempDataset_<%=cid%> = <%=tempDataset%>;
|
||||
<% if(isLog4jEnabled){ %>
|
||||
log.info("<%=cid%> - temporary Dataset name : " + tempDataset_<%=cid%>);
|
||||
<%
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
com.google.cloud.bigquery.QueryJobConfiguration jobConfDryRun_<%=cid%> = queryConfiguration_<%=cid%> .setDryRun(true).build();
|
||||
com.google.cloud.bigquery.Job jobDryRun_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.of(jobConfDryRun_<%=cid%>));
|
||||
|
||||
String queryLocation_<%=cid%> =jobDryRun_<%=cid%>.getJobId().getLocation();
|
||||
String location_<%=cid%> = queryLocation_<%=cid%> == null ? "US" : queryLocation_<%=cid%>;
|
||||
String tempDataset_<%=cid%> = java.util.UUID.randomUUID().toString().replaceAll("-", "")
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt())
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
|
||||
|
||||
<% if(isLog4jEnabled){ %>
|
||||
log.info("<%=cid%> - query location :" + queryLocation_<%=cid%>);
|
||||
log.info("<%=cid%> - temporary dataset location :" + location_<%=cid%>);
|
||||
log.info("<%=cid%> - temporary Dataset name : " + tempDataset_<%=cid%>);
|
||||
|
||||
<% } %>
|
||||
|
||||
com.google.cloud.bigquery.DatasetInfo datasetInfo_<%=cid%> = com.google.cloud.bigquery.DatasetInfo.newBuilder(tempDataset_<%=cid%>).setLocation(location_<%=cid%>).build();
|
||||
com.google.cloud.bigquery.Dataset dataset_<%=cid%> = bigquery_<%=cid%> .create(datasetInfo_<%=cid%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
String tempTable_<%=cid%> = java.util.UUID.randomUUID().toString().replaceAll("-", "")
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt())
|
||||
+ Integer.toHexString(java.util.concurrent.ThreadLocalRandom.current().nextInt());
|
||||
<% if(isLog4jEnabled){ %>
|
||||
log.info("<%=cid%> - temporary table name : " + tempTable_<%=cid%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
queryConfiguration_<%=cid%>
|
||||
.setDryRun(false)
|
||||
.setAllowLargeResults(true)
|
||||
.setDestinationTable(com.google.cloud.bigquery.TableId.of(tempDataset_<%=cid%>, tempTable_<%=cid%>));
|
||||
|
||||
com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId
|
||||
.newBuilder().setProject(<%=projectId%>)
|
||||
.setJob(java.util.UUID.randomUUID().toString())
|
||||
.build();
|
||||
|
||||
<% if(isLog4jEnabled){ %>
|
||||
log.info("<%=cid%> - job location : " + jobId_<%=cid%>.getLocation());
|
||||
<% } %>
|
||||
<% } else { %>
|
||||
com.google.cloud.bigquery.JobId jobId_<%=cid%> = com.google.cloud.bigquery.JobId.of(<%=projectId%>,java.util.UUID.randomUUID().toString());
|
||||
|
||||
<% } %>
|
||||
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.newBuilder(queryConfiguration_<%=cid%>.build()).setJobId(jobId_<%=cid%>).build());
|
||||
|
||||
|
||||
<% if(isLog4jEnabled) { %>
|
||||
log.info("<%=cid%> - Sending job " + jobId_<%=cid%> + " with query: " + <%=query%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<% } %>
|
||||
|
||||
job_<%=cid%> = job_<%=cid%>.waitFor();
|
||||
|
||||
|
||||
if (job_<%=cid%> == null) {
|
||||
throw new RuntimeException("Job no longer exists");
|
||||
} else if (job_<%=cid%>.getStatus().getError() != null) {
|
||||
throw new RuntimeException(job_<%=cid%>.getStatus().getError().toString());
|
||||
}
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
<% if(isLog4jEnabled) { %>
|
||||
log.info("<%=cid%> - Job " + jobId_<%=cid%> + " finished successfully.");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<% } %>
|
||||
com.google.cloud.bigquery.TableResult result_<%=cid%> = job_<%=cid%>.getQueryResults();
|
||||
long nb_line_<%=cid%> = 0;
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
if (authMode.equals("OAUTH")) {
|
||||
%>
|
||||
while (true) {
|
||||
@@ -286,13 +328,13 @@
|
||||
insert_<%=cid %>.getConfiguration().getQuery()
|
||||
.getDestinationTable().getTableId())
|
||||
.setPageToken(pageToken_<%=cid%>).execute();
|
||||
|
||||
|
||||
List<com.google.api.services.bigquery.model.TableRow> rows_<%=cid %> = dataList_<%=cid %>.getRows();
|
||||
if(rows_<%=cid %> == null) {
|
||||
// Means there is no record.
|
||||
rows_<%=cid %> = new java.util.ArrayList<com.google.api.services.bigquery.model.TableRow>();
|
||||
}
|
||||
|
||||
|
||||
for (com.google.api.services.bigquery.model.TableRow row_<%=cid %> : rows_<%=cid %>) {
|
||||
java.util.List<com.google.api.services.bigquery.model.TableCell> field_<%=cid %> = row_<%=cid %>.getF();
|
||||
Object value_<%=cid%> = null;
|
||||
@@ -322,16 +364,16 @@
|
||||
boolean advancedSeparator = (advancedSeparatorStr!=null&&!("").equals(advancedSeparatorStr))?("true").equals(advancedSeparatorStr):false;
|
||||
String thousandsSeparator = ElementParameterParser.getValueWithJavaType(node, "__THOUSANDS_SEPARATOR__", JavaTypesManager.CHARACTER);
|
||||
String decimalSeparator = ElementParameterParser.getValueWithJavaType(node, "__DECIMAL_SEPARATOR__", JavaTypesManager.CHARACTER);
|
||||
|
||||
|
||||
List<IMetadataColumn> columns = metadata.getListColumns();
|
||||
int nbColumns = columns.size();
|
||||
for (int i = 0; i < nbColumns; i++ ) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
String columnName = column.getLabel();
|
||||
|
||||
|
||||
String typeToGenerate = JavaTypesManager.getTypeToGenerate(column.getTalendType(), column.isNullable());
|
||||
JavaType javaType = JavaTypesManager.getJavaTypeFromId(column.getTalendType());
|
||||
String patternValue = column.getPattern() == null || column.getPattern().trim().length() == 0 ? null : column.getPattern();
|
||||
String patternValue = column.getPattern() == null || column.getPattern().trim().length() == 0 ? null : column.getPattern();
|
||||
if (authMode.equals("OAUTH")) {
|
||||
%>
|
||||
value_<%=cid%> = field_<%=cid %>.get(<%=i%>).getV();
|
||||
@@ -345,34 +387,34 @@
|
||||
}
|
||||
%>
|
||||
if(com.google.api.client.util.Data.isNull(value_<%=cid%>)) value_<%=cid%> = null;
|
||||
|
||||
|
||||
if(value_<%=cid%> != null){
|
||||
|
||||
|
||||
<%
|
||||
if (javaType == JavaTypesManager.STRING) {
|
||||
%>
|
||||
<%=connName%>.<%=columnName%> = value_<%=cid%>.toString();
|
||||
<%
|
||||
} else if (javaType == JavaTypesManager.OBJECT) {
|
||||
%>
|
||||
%>
|
||||
<%=connName%>.<%=columnName%> = value_<%=cid%>;
|
||||
<%
|
||||
} else if(javaType == JavaTypesManager.DATE) {
|
||||
%>
|
||||
|
||||
<%
|
||||
} else if(javaType == JavaTypesManager.DATE) {
|
||||
%>
|
||||
|
||||
if (value_<%=cid%>.toString().contains("-")) {
|
||||
<%=connName%>.<%=columnName%> = ParserUtils.parseTo_Date(value_<%=cid%>.toString(),<%=patternValue%>);
|
||||
} else {
|
||||
<%=connName%>.<%=columnName%> = ParserUtils.parseTo_Date(value_<%=cid%>.toString(),<%=patternValue%>);
|
||||
} else {
|
||||
<%=connName%>.<%=columnName%> = ParserUtils.parseTo_Date(value_<%=cid%>.toString());
|
||||
}
|
||||
|
||||
<%
|
||||
} else if(advancedSeparator && JavaTypesManager.isNumberType(javaType)) {
|
||||
%>
|
||||
|
||||
<%
|
||||
} else if(advancedSeparator && JavaTypesManager.isNumberType(javaType)) {
|
||||
%>
|
||||
<%=connName%>.<%=columnName%> = ParserUtils.parseTo_<%= typeToGenerate %>(ParserUtils.parseTo_Number(value_<%=cid%>.toString(), <%= thousandsSeparator %>, <%= decimalSeparator %>));
|
||||
<%
|
||||
} else if(javaType == JavaTypesManager.BYTE_ARRAY) {
|
||||
%>
|
||||
} else if(javaType == JavaTypesManager.BYTE_ARRAY) {
|
||||
%>
|
||||
<%=connName%>.<%=columnName%> = value_<%=cid%>.toString().getBytes(<%=encoding %>);
|
||||
<%
|
||||
} else {
|
||||
|
||||
@@ -1,16 +1,20 @@
|
||||
<%@ jet
|
||||
<%@ jet
|
||||
imports="
|
||||
org.talend.core.model.process.INode
|
||||
org.talend.core.model.process.INode
|
||||
org.talend.designer.codegen.config.CodeGeneratorArgument
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
"
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
"
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/Log4j/Log4jFileUtil.javajet"%>
|
||||
<%
|
||||
CodeGeneratorArgument codeGenArgument = (CodeGeneratorArgument) argument;
|
||||
INode node = (INode)codeGenArgument.getArgument();
|
||||
String cid = node.getUniqueName();
|
||||
String authMode = ElementParameterParser.getValue(node,"__AUTH_MODE__");
|
||||
String authMode = ElementParameterParser.getValue(node,"__AUTH_MODE__");
|
||||
String projectId = ElementParameterParser.getValue(node,"__PROJECT_ID__");
|
||||
String resultSizeType = ElementParameterParser.getValue(node,"__RESULT_SIZE__");
|
||||
boolean isCustomTemporaryName = ElementParameterParser.getBooleanValue(node,"__USE_CUSTOM_TEMPORARY_DATASET__");
|
||||
|
||||
if (authMode.equals("OAUTH")) {
|
||||
%>
|
||||
}
|
||||
@@ -24,10 +28,20 @@ imports="
|
||||
} else if (authMode.equals("SERVICEACCOUNT")) {
|
||||
%>
|
||||
}
|
||||
bigquery_<%=cid%>.delete(com.google.cloud.bigquery.TableId.of(dataset, temp_table));
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
}
|
||||
log4jFileUtil.retrievedDataNumberInfo(node);
|
||||
%>
|
||||
<% if (resultSizeType.equals("LARGE") || resultSizeType.equals("AUTO")) {
|
||||
if (isCustomTemporaryName) {
|
||||
%>
|
||||
bigquery_<%=cid%>.delete(com.google.cloud.bigquery.TableId.of(tempDataset_<%=cid%>, tempTable_<%=cid%>));
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
com.google.cloud.bigquery.DatasetId datasetId_<%=cid%> = com.google.cloud.bigquery.DatasetId.of(<%=projectId%>, tempDataset_<%=cid%>);
|
||||
bigquery_<%=cid%>.delete(datasetId_<%=cid%>, com.google.cloud.bigquery.BigQuery.DatasetDeleteOption.deleteContents());
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
}
|
||||
log4jFileUtil.retrievedDataNumberInfo(node);
|
||||
%>
|
||||
|
||||
@@ -157,6 +157,17 @@
|
||||
REQUIRED="true" REPOSITORY_VALUE="ENCODING">
|
||||
<DEFAULT>"ISO-8859-15"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="USE_CUSTOM_TEMPORARY_DATASET" FIELD="CHECK" NUM_ROW="30"
|
||||
SHOW_IF="(RESULT_SIZE != 'SMALL')">
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="TEMPORARY_DATASET_NAME" FIELD="TEXT" REQUIRED="true" NUM_ROW="40"
|
||||
SHOW_IF="(USE_CUSTOM_TEMPORARY_DATASET == 'true') AND (RESULT_SIZE != 'SMALL')">
|
||||
<DEFAULT>"temp_dataset"</DEFAULT>
|
||||
</PARAMETER>
|
||||
</ADVANCED_PARAMETERS>
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
@@ -167,7 +178,7 @@
|
||||
<IMPORT NAME="google-oauth-client-1.19.0.jar" MODULE="google-oauth-client-1.19.0.jar" MVN="mvn:org.talend.libraries/google-oauth-client-1.19.0/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-oauth-client-1.19.0.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-jackson2-1.19.0.jar" MODULE="google-http-client-jackson2-1.19.0.jar" MVN="mvn:org.talend.libraries/google-http-client-jackson2-1.19.0/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.google/lib/google-http-client-jackson2-1.19.0.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="guava-jdk5-13.0.jar" MODULE="guava-jdk5-13.0.jar" MVN="mvn:org.talend.libraries/guava-jdk5-13.0/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.guava/lib/guava-jdk5-13.0.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.9.9.jar" MODULE="jackson-core-2.9.9.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.9" REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1" REQUIRED="true" />
|
||||
<IMPORT NAME="google-cloud-bigquery-1.32.0.jar" MODULE="google-cloud-bigquery-1.32.0.jar" MVN="mvn:com.google.cloud/google-cloud-bigquery/1.32.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="google-http-client-jackson-1.23.0.jar" MODULE="google-http-client-jackson-1.23.0.jar" MVN="mvn:com.google.http-client/google-http-client-jackson/1.23.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="threetenbp-1.3.3.jar" MODULE="threetenbp-1.3.3.jar" MVN="mvn:org.threeten/threetenbp/1.3.3" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
|
||||
@@ -27,5 +27,6 @@ RESULT_SIZE.NAME=Result size
|
||||
RESULT_SIZE.ITEM.SMALL=Small(without allowLargeResults)
|
||||
RESULT_SIZE.ITEM.LARGE=Large(with allowLargeResults)
|
||||
RESULT_SIZE.ITEM.AUTO=Auto
|
||||
|
||||
USE_LEGACY_SQL.NAME=Use Legacy SQL
|
||||
USE_CUSTOM_TEMPORARY_DATASET.NAME=Use custom temporary Dataset name
|
||||
TEMPORARY_DATASET_NAME.NAME=Temporary Dataset name
|
||||
USE_LEGACY_SQL.NAME=Use Legacy SQL
|
||||
|
||||
@@ -195,8 +195,8 @@
|
||||
<IMPORT NAME="guava-jdk5-13.0.jar" MODULE="guava-jdk5-13.0.jar"
|
||||
MVN="mvn:org.talend.libraries/guava-jdk5-13.0/6.0.0"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.guava/lib/guava-jdk5-13.0.jar" REQUIRED="true"/>
|
||||
<IMPORT NAME="jackson-core-2.9.9.jar" MODULE="jackson-core-2.9.9.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.9" REQUIRED="true"/>
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1" REQUIRED="true"/>
|
||||
<IMPORT NAME="google-cloud-bigquery-1.32.0.jar" MODULE="google-cloud-bigquery-1.32.0.jar"
|
||||
MVN="mvn:com.google.cloud/google-cloud-bigquery/1.32.0"
|
||||
REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'"/>
|
||||
|
||||
@@ -72,13 +72,13 @@
|
||||
MODULE="boxjavalibv2-v3.0.10.jar" MVN="mvn:org.talend.libraries/boxjavalibv2-v3.0.10/6.0.0"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core"
|
||||
MODULE="jackson-core-2.9.9.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.9"
|
||||
MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind"
|
||||
MODULE="jackson-databind-2.9.9.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.9.9"
|
||||
MODULE="jackson-databind-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations"
|
||||
MODULE="jackson-annotations-2.9.0.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.9.0"
|
||||
MODULE="jackson-annotations-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="apache-http-client"
|
||||
MODULE="httpclient-4.2.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.2.5/6.0.0"
|
||||
|
||||
@@ -127,13 +127,13 @@
|
||||
MODULE="boxjavalibv2-v3.0.10.jar" MVN="mvn:org.talend.libraries/boxjavalibv2-v3.0.10/6.0.0"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-core"
|
||||
MODULE="jackson-core-2.9.9.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.9"
|
||||
MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-databind"
|
||||
MODULE="jackson-databind-2.9.9.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.9.9"
|
||||
MODULE="jackson-databind-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-annotations"
|
||||
MODULE="jackson-annotations-2.9.0.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.9.0"
|
||||
MODULE="jackson-annotations-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-client"
|
||||
MODULE="httpclient-4.2.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.2.5/6.0.0"
|
||||
|
||||
@@ -88,13 +88,13 @@
|
||||
MODULE="boxjavalibv2-v3.0.10.jar" MVN="mvn:org.talend.libraries/boxjavalibv2-v3.0.10/6.0.0"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-core"
|
||||
MODULE="jackson-core-2.9.9.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.9"
|
||||
MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-databind"
|
||||
MODULE="jackson-databind-2.9.9.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.9.9"
|
||||
MODULE="jackson-databind-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-annotations"
|
||||
MODULE="jackson-annotations-2.9.0.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.9.0"
|
||||
MODULE="jackson-annotations-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-client"
|
||||
MODULE="httpclient-4.2.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.2.5/6.0.0"
|
||||
|
||||
@@ -98,13 +98,13 @@
|
||||
MODULE="boxjavalibv2-v3.0.10.jar" MVN="mvn:org.talend.libraries/boxjavalibv2-v3.0.10/6.0.0"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-core"
|
||||
MODULE="jackson-core-2.9.9.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.9"
|
||||
MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-databind"
|
||||
MODULE="jackson-databind-2.9.9.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.9.9"
|
||||
MODULE="jackson-databind-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-annotations"
|
||||
MODULE="jackson-annotations-2.9.0.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.9.0"
|
||||
MODULE="jackson-annotations-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-client"
|
||||
MODULE="httpclient-4.2.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.2.5/6.0.0"
|
||||
|
||||
@@ -106,13 +106,13 @@
|
||||
MODULE="boxjavalibv2-v3.0.10.jar" MVN="mvn:org.talend.libraries/boxjavalibv2-v3.0.10/6.0.0"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-core"
|
||||
MODULE="jackson-core-2.9.9.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.9.9"
|
||||
MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-databind"
|
||||
MODULE="jackson-databind-2.9.9.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.9.9"
|
||||
MODULE="jackson-databind-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-annotations"
|
||||
MODULE="jackson-annotations-2.9.0.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.9.0"
|
||||
MODULE="jackson-annotations-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-client"
|
||||
MODULE="httpclient-4.2.5.jar" MVN="mvn:org.talend.libraries/httpclient-4.2.5/6.0.0"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user