Compare commits
225 Commits
cmeng/patc
...
patch/TPS-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6758a3424b | ||
|
|
11a41a331e | ||
|
|
e9fa81a1c8 | ||
|
|
9153d30f6e | ||
|
|
dd863cfd15 | ||
|
|
55d48cfe91 | ||
|
|
7b325e8707 | ||
|
|
7465b41a34 | ||
|
|
79fb201844 | ||
|
|
45edbf18a1 | ||
|
|
f347a16522 | ||
|
|
b9e4faf2bd | ||
|
|
1ab7eaeca6 | ||
|
|
069a7b26c3 | ||
|
|
6a3651d3b6 | ||
|
|
029c0ccb5c | ||
|
|
1382b4efb4 | ||
|
|
b0d3a70cf3 | ||
|
|
1f20471fbe | ||
|
|
d28ffc5736 | ||
|
|
146cf1dc0e | ||
|
|
6c03ca588a | ||
|
|
b528ca9a0d | ||
|
|
888765461c | ||
|
|
ae2f00f1a8 | ||
|
|
704b63d59c | ||
|
|
a92fedb9b4 | ||
|
|
f1a7d2f235 | ||
|
|
1702b27493 | ||
|
|
cf55315820 | ||
|
|
d25b125c08 | ||
|
|
290d9566ed | ||
|
|
0c85bdc4be | ||
|
|
a4cb0d13d2 | ||
|
|
502742bad2 | ||
|
|
83a64d3d2c | ||
|
|
30bbb27e87 | ||
|
|
b554d94736 | ||
|
|
13bbfcca1d | ||
|
|
6bebbba8ee | ||
|
|
37dc7ca816 | ||
|
|
242fa6c729 | ||
|
|
d1d4dcd7f6 | ||
|
|
885d14671a | ||
|
|
8fa189bd31 | ||
|
|
fe86a1ef43 | ||
|
|
8751efe56e | ||
|
|
236fb7fc65 | ||
|
|
c55409088f | ||
|
|
7ead0130ac | ||
|
|
562d3700b1 | ||
|
|
3d5fb83763 | ||
|
|
41f1df71d7 | ||
|
|
0f7596dcb8 | ||
|
|
520e379e48 | ||
|
|
28ae35dec6 | ||
|
|
21245a297a | ||
|
|
5131d00675 | ||
|
|
f256fc6f38 | ||
|
|
30fa41dd46 | ||
|
|
8a3a1a0289 | ||
|
|
b3468eba73 | ||
|
|
9d9d6b135b | ||
|
|
7cbcd2967b | ||
|
|
918ca363fa | ||
|
|
a473b624b7 | ||
|
|
1f6128dfd7 | ||
|
|
6fa4c42df1 | ||
|
|
1e61474fc0 | ||
|
|
d6242e11cd | ||
|
|
d007d3740a | ||
|
|
765accb8a3 | ||
|
|
b51c1282f7 | ||
|
|
c98e18a58c | ||
|
|
e576293e58 | ||
|
|
55812da6b6 | ||
|
|
074c13dd4e | ||
|
|
6088e2c2ca | ||
|
|
1637e6220c | ||
|
|
e96d2af073 | ||
|
|
a1e453e79a | ||
|
|
c8c4a586d0 | ||
|
|
89435124fe | ||
|
|
ae8e41f36b | ||
|
|
e6fd8b0614 | ||
|
|
f4223f7a2b | ||
|
|
64dcafb080 | ||
|
|
402fe1ffbc | ||
|
|
02c66a7e93 | ||
|
|
207d1c9635 | ||
|
|
a4d0adb671 | ||
|
|
0ce6a06f8e | ||
|
|
15cbbf362c | ||
|
|
c139b0893d | ||
|
|
4c2e419bc0 | ||
|
|
adc91e4169 | ||
|
|
09607ed581 | ||
|
|
0a5b925dc4 | ||
|
|
414ab39434 | ||
|
|
a269f74a30 | ||
|
|
530814c490 | ||
|
|
1da69fb285 | ||
|
|
3c00488dc8 | ||
|
|
17f54191cf | ||
|
|
ed99155812 | ||
|
|
2276f4b51a | ||
|
|
da5744d1e5 | ||
|
|
3c04002b5e | ||
|
|
8b67961ade | ||
|
|
ec914f50fe | ||
|
|
9ab7f01201 | ||
|
|
af79e71c25 | ||
|
|
c50e437c59 | ||
|
|
ca07dd16cf | ||
|
|
64794a596c | ||
|
|
ff595fd205 | ||
|
|
c035091f88 | ||
|
|
3d8c28928a | ||
|
|
54af03d3ef | ||
|
|
1adad9fe93 | ||
|
|
a20ea15f4d | ||
|
|
1df79674e1 | ||
|
|
c0fedf4ef4 | ||
|
|
f3b45bf229 | ||
|
|
243a1f3326 | ||
|
|
4abf245ca1 | ||
|
|
d71b4b148a | ||
|
|
45f94c22be | ||
|
|
1c41f0c05d | ||
|
|
d70ad09a49 | ||
|
|
6e440ed726 | ||
|
|
d8ace9d577 | ||
|
|
8b3bbf7dcb | ||
|
|
0d0a67be0c | ||
|
|
3e410b5373 | ||
|
|
aad3651c38 | ||
|
|
6515a196e2 | ||
|
|
cdc51a076c | ||
|
|
5e327d255d | ||
|
|
63fffa6d6c | ||
|
|
9a974a67c0 | ||
|
|
2a8a6c074a | ||
|
|
53435511fb | ||
|
|
7840e1783e | ||
|
|
48c26b2d21 | ||
|
|
8027965b4c | ||
|
|
e9bad7dd32 | ||
|
|
593a717084 | ||
|
|
fb884595dd | ||
|
|
756ba629ed | ||
|
|
aac496bf8f | ||
|
|
7ac98d48a3 | ||
|
|
8752a75abd | ||
|
|
9e85699b85 | ||
|
|
c09016f435 | ||
|
|
028b6681c3 | ||
|
|
22fbf0ff0b | ||
|
|
ec74ac0f35 | ||
|
|
c5d44ab48b | ||
|
|
1db9854428 | ||
|
|
5d1ab30b34 | ||
|
|
3502a8e79a | ||
|
|
f3b91a3cac | ||
|
|
e1bceeea2d | ||
|
|
967a3d94ce | ||
|
|
69a5234730 | ||
|
|
c8f4dd2e6d | ||
|
|
a0b0366bcb | ||
|
|
f24633190e | ||
|
|
8e4e04c9cd | ||
|
|
05a0815778 | ||
|
|
965c02c58d | ||
|
|
5eb4100d98 | ||
|
|
b0059fdb12 | ||
|
|
061f3dc431 | ||
|
|
e85afa939c | ||
|
|
cb475f4e5e | ||
|
|
a76b52f3b9 | ||
|
|
4689d45132 | ||
|
|
9b5eccc67c | ||
|
|
53d4b392bc | ||
|
|
a2510f5e2a | ||
|
|
539710015b | ||
|
|
a398a56e95 | ||
|
|
ba647bde38 | ||
|
|
3b37b58fd0 | ||
|
|
cc07722ebb | ||
|
|
625792e472 | ||
|
|
b55e6c1c02 | ||
|
|
b43b149ba4 | ||
|
|
918cf4eed5 | ||
|
|
5fcaae4e48 | ||
|
|
41231a43b6 | ||
|
|
03eeaac507 | ||
|
|
ad7316b2ce | ||
|
|
31e27f97e0 | ||
|
|
09fd069243 | ||
|
|
f5ea29812e | ||
|
|
5acbcd229e | ||
|
|
203187c05f | ||
|
|
39996404a7 | ||
|
|
50e7057eb9 | ||
|
|
94c39f79ff | ||
|
|
a1b85f19c1 | ||
|
|
66755538e8 | ||
|
|
a2209548ae | ||
|
|
d8e604bdb0 | ||
|
|
e4194eea26 | ||
|
|
3a9ea6fafd | ||
|
|
d71fce6268 | ||
|
|
11368d3910 | ||
|
|
d01477ab11 | ||
|
|
7900e44e13 | ||
|
|
04988ed52b | ||
|
|
e0b54b7df4 | ||
|
|
9b70682a48 | ||
|
|
75fb043c1c | ||
|
|
e8c2b70985 | ||
|
|
e727191389 | ||
|
|
340b59a17e | ||
|
|
6a2257f107 | ||
|
|
d4751f4cfb | ||
|
|
9e9406e2fa | ||
|
|
647dbe0676 | ||
|
|
775e5a59f6 |
57
PATCH_RELEASE_NOTE.md
Normal file
57
PATCH_RELEASE_NOTE.md
Normal file
@@ -0,0 +1,57 @@
|
||||
---
|
||||
version: 7.3.1
|
||||
module: https://talend.poolparty.biz/coretaxonomy/42
|
||||
product:
|
||||
- https://talend.poolparty.biz/coretaxonomy/23
|
||||
---
|
||||
|
||||
# TPS-4723
|
||||
|
||||
| Info | Value |
|
||||
| ---------------- | ---------------- |
|
||||
| Patch Name | Patch\_20210304\_TPS-4723\_v1-7.3.1 |
|
||||
| Release Date | 2021-03-04 |
|
||||
| Target Version | 20200219_1130-V7.3.1 |
|
||||
| Product affected | Talend Studio |
|
||||
|
||||
## Introduction
|
||||
|
||||
This is a self-contained patch.
|
||||
|
||||
**NOTE**: For information on how to obtain this patch, reach out to your Support contact at Talend.
|
||||
|
||||
## Fixed issues
|
||||
|
||||
This patch contains the following fixes:
|
||||
|
||||
- TPS-4723 [7.3.1] Getting Permission denied error in tFileInputExcel in Talend (TDI-45561)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Consider the following requirements for your system:
|
||||
|
||||
- Talend Studio 7.3.1 and Patch_20200529_R2020-05_v1-7.3.1 must be installed.
|
||||
|
||||
## Installation
|
||||
|
||||
### Installing the patch using Software update
|
||||
|
||||
1) Logon TAC and switch to Configuration->Software Update, then enter the correct values and save referring to the documentation: https://help.talend.com/reader/f7Em9WV_cPm2RRywucSN0Q/j9x5iXV~vyxMlUafnDejaQ
|
||||
|
||||
2) Switch to Software update page, where the new patch will be listed. The patch can be downloaded from here into the nexus repository.
|
||||
|
||||
3) On Studio Side: Logon Studio with remote mode, on the logon page the Update button is displayed: click this button to install the patch.
|
||||
|
||||
### Installing the patch using Talend Studio
|
||||
|
||||
1) Create a folder named "patches" under your studio installer directory and copy the patch .zip file to this folder.
|
||||
|
||||
2) Restart your studio: a window pops up, then click OK to install the patch, or restart the commandline and the patch will be installed automatically.
|
||||
|
||||
### Installing the patch using Commandline
|
||||
|
||||
Execute the following commands:
|
||||
|
||||
1. Talend-Studio-win-x86_64.exe -nosplash -application org.talend.commandline.CommandLine -consoleLog -data commandline-workspace startServer -p 8002 --talendDebug
|
||||
2. initRemote {tac_url} -ul {TAC login username} -up {TAC login password}
|
||||
3. checkAndUpdate -tu {TAC login username} -tup {TAC login password}
|
||||
@@ -16,11 +16,9 @@
|
||||
</requires>
|
||||
<plugin id="org.talend.libraries.apache" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.axis2" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.batik" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.chemistry" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.common" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.cxf" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.google" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.http" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.lucene" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.xml" download-size="0" install-size="0" version="0.0.0"/>
|
||||
|
||||
@@ -91,10 +91,10 @@ if((codePart.equals(ECodePart.END))&&(stat || logstashCurrent)){
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel.trim());
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel.trim());
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
@@ -125,10 +125,10 @@ if((codePart.equals(ECodePart.END))&&(stat || logstashCurrent)){
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel.trim());
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel.trim());
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
|
||||
@@ -197,10 +197,10 @@
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel.trim());
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel.trim());
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
%>
|
||||
@@ -233,10 +233,10 @@
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel.trim());
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel.trim());
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
%>
|
||||
@@ -260,10 +260,10 @@
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel.trim());
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel.trim());
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
|
||||
@@ -296,10 +296,10 @@
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel.trim());
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel.trim());
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
|
||||
@@ -435,7 +435,7 @@
|
||||
if(logstashCurrent) {
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
String label = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String nodeLabel = ((label==null || "__UNIQUE_NAME__".equals(label) || label.contains("\"")) ? node.getUniqueName() : label);
|
||||
String nodeLabel = ((label==null || "__UNIQUE_NAME__".equals(label) || label.contains("\"")) ? node.getUniqueName() : label.trim());
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
<%=jobStructureCatcher.getUniqueName() %>.addCM("<%=node.getUniqueName()%>", "<%=nodeLabel%>", "<%=node.getComponent().getName()%>");
|
||||
|
||||
@@ -642,6 +642,16 @@
|
||||
}
|
||||
%>
|
||||
|
||||
boolean inOSGi = routines.system.BundleUtils.inOSGi();
|
||||
|
||||
if (inOSGi) {
|
||||
java.util.Dictionary<String, Object> jobProperties = routines.system.BundleUtils.getJobProperties(jobName);
|
||||
|
||||
if (jobProperties != null) {
|
||||
contextStr = (String)jobProperties.get("context");
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
//call job/subjob with an existing context, like: --context=production. if without this parameter, there will use the default context instead.
|
||||
java.io.InputStream inContext = <%=className%>.class.getClassLoader().getResourceAsStream("<%=jobClassPackageFolder%>/contexts/" + contextStr + ".properties");
|
||||
@@ -649,13 +659,15 @@
|
||||
inContext = <%=className%>.class.getClassLoader().getResourceAsStream("config/contexts/" + contextStr + ".properties");
|
||||
}
|
||||
if (inContext != null) {
|
||||
//defaultProps is in order to keep the original context value
|
||||
if(context != null && context.isEmpty()) {
|
||||
try {
|
||||
//defaultProps is in order to keep the original context value
|
||||
if(context != null && context.isEmpty()) {
|
||||
defaultProps.load(inContext);
|
||||
context = new ContextProperties(defaultProps);
|
||||
}
|
||||
} finally {
|
||||
inContext.close();
|
||||
}
|
||||
|
||||
inContext.close();
|
||||
} else if (!isDefaultContext) {
|
||||
//print info and job continue to run, for case: context_param is not empty.
|
||||
System.err.println("Could not find the context " + contextStr);
|
||||
@@ -1194,6 +1206,24 @@ if (execStat) {
|
||||
}
|
||||
%>
|
||||
int returnCode = 0;
|
||||
|
||||
<%
|
||||
if (isRunInMultiThread) {
|
||||
%>
|
||||
Integer localErrorCode = (Integer)(((java.util.Map)threadLocal.get()).get("errorCode"));
|
||||
String localStatus = (String)(((java.util.Map)threadLocal.get()).get("status"));
|
||||
if (localErrorCode != null) {
|
||||
if (errorCode == null || localErrorCode.compareTo(errorCode) > 0) {
|
||||
errorCode = localErrorCode;
|
||||
}
|
||||
}
|
||||
if (localStatus != null && !status.equals("failure")){
|
||||
status = localStatus;
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
if(errorCode == null) {
|
||||
returnCode = status != null && status.equals("failure") ? 1 : 0;
|
||||
} else {
|
||||
|
||||
@@ -60,6 +60,10 @@ if ((metadatas != null) && (metadatas.size() > 0)) { // metadata
|
||||
|
||||
org.talend.components.api.component.ComponentDefinition def_<%=cid %> =
|
||||
new <%= def.getClass().getName()%>();
|
||||
|
||||
org.talend.components.api.component.runtime.Writer writer_<%=cid%> = null;
|
||||
org.talend.components.api.component.runtime.Reader reader_<%=cid%> = null;
|
||||
|
||||
<%
|
||||
List<Component.CodegenPropInfo> propsToProcess = component.getCodegenPropInfos(componentProps);
|
||||
%>
|
||||
@@ -240,11 +244,11 @@ if(isTopologyNone) {
|
||||
|
||||
if (hasOutputOnly || asInputComponent) {
|
||||
%>
|
||||
org.talend.components.api.component.runtime.Source source_<%=cid%> =
|
||||
(org.talend.components.api.component.runtime.Source)sourceOrSink_<%=cid%>;
|
||||
org.talend.components.api.component.runtime.Reader reader_<%=cid%> =
|
||||
source_<%=cid%>.createReader(container_<%=cid%>);
|
||||
reader_<%=cid%> = new org.talend.codegen.flowvariables.runtime.FlowVariablesReader(reader_<%=cid%>, container_<%=cid%>);
|
||||
if (sourceOrSink_<%=cid%> instanceof org.talend.components.api.component.runtime.Source) {
|
||||
org.talend.components.api.component.runtime.Source source_<%=cid%> =
|
||||
(org.talend.components.api.component.runtime.Source)sourceOrSink_<%=cid%>;
|
||||
reader_<%=cid%> = source_<%=cid%>.createReader(container_<%=cid%>);
|
||||
reader_<%=cid%> = new org.talend.codegen.flowvariables.runtime.FlowVariablesReader(reader_<%=cid%>, container_<%=cid%>);
|
||||
|
||||
<%
|
||||
IConnection main = null;
|
||||
@@ -266,19 +270,19 @@ if (hasOutputOnly || asInputComponent) {
|
||||
IConnection schemaSourceConnector = (main!=null ? main : reject);
|
||||
String schemaSourceConnectorName = schemaSourceConnector.getMetadataTable().getAttachedConnector();
|
||||
%>
|
||||
boolean multi_output_is_allowed_<%=cid%> = false;
|
||||
boolean multi_output_is_allowed_<%=cid%> = false;
|
||||
<% //take care SourceOrSink.validate will change the schema if it contains include-all-fields, so need to get design Avro schema before validate %>
|
||||
org.talend.components.api.component.Connector c_<%=cid%> = null;
|
||||
for (org.talend.components.api.component.Connector currentConnector : props_<%=cid %>.getAvailableConnectors(null, true)) {
|
||||
if (currentConnector.getName().equals("<%=schemaSourceConnectorName%>")) {
|
||||
c_<%=cid%> = currentConnector;
|
||||
}
|
||||
org.talend.components.api.component.Connector c_<%=cid%> = null;
|
||||
for (org.talend.components.api.component.Connector currentConnector : props_<%=cid %>.getAvailableConnectors(null, true)) {
|
||||
if (currentConnector.getName().equals("<%=schemaSourceConnectorName%>")) {
|
||||
c_<%=cid%> = currentConnector;
|
||||
}
|
||||
|
||||
if (currentConnector.getName().equals("REJECT")) {//it's better to move the code to javajet
|
||||
multi_output_is_allowed_<%=cid%> = true;
|
||||
if (currentConnector.getName().equals("REJECT")) {//it's better to move the code to javajet
|
||||
multi_output_is_allowed_<%=cid%> = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
org.apache.avro.Schema schema_<%=cid%> = props_<%=cid %>.getSchema(c_<%=cid%>, true);
|
||||
org.apache.avro.Schema schema_<%=cid%> = props_<%=cid %>.getSchema(c_<%=cid%>, true);
|
||||
|
||||
<%
|
||||
irToRow = new IndexedRecordToRowStructGenerator(cid, null, columnList);
|
||||
@@ -286,117 +290,119 @@ if (hasOutputOnly || asInputComponent) {
|
||||
}
|
||||
%>
|
||||
|
||||
// Iterate through the incoming data.
|
||||
boolean available_<%=cid%> = reader_<%=cid%>.start();
|
||||
// Iterate through the incoming data.
|
||||
boolean available_<%=cid%> = reader_<%=cid%>.start();
|
||||
|
||||
resourceMap.put("reader_<%=cid%>", reader_<%=cid%>);
|
||||
resourceMap.put("reader_<%=cid%>", reader_<%=cid%>);
|
||||
|
||||
for (; available_<%=cid%>; available_<%=cid%> = reader_<%=cid%>.advance()) {
|
||||
nb_line_<%=cid %>++;
|
||||
for (; available_<%=cid%>; available_<%=cid%> = reader_<%=cid%>.advance()) {
|
||||
nb_line_<%=cid %>++;
|
||||
|
||||
<%if(hasDataOutput) {%>
|
||||
if (multi_output_is_allowed_<%=cid%>) {
|
||||
<%if(main!=null){%>
|
||||
<%=main.getName()%> = null;
|
||||
<%}%>
|
||||
|
||||
<%if(reject!=null){%>
|
||||
<%=reject.getName()%> = null;
|
||||
<%}%>
|
||||
}
|
||||
<%}%>
|
||||
|
||||
<%if(hasDataOutput) {%>
|
||||
if (multi_output_is_allowed_<%=cid%>) {
|
||||
<%if(main!=null){%>
|
||||
<%=main.getName()%> = null;
|
||||
<%}%>
|
||||
|
||||
<%if(reject!=null){%>
|
||||
<%=reject.getName()%> = null;
|
||||
<%}%>
|
||||
}
|
||||
<%}%>
|
||||
|
||||
try {
|
||||
Object data_<%=cid%> = reader_<%=cid%>.getCurrent();
|
||||
<%
|
||||
if (main != null) {
|
||||
%>
|
||||
|
||||
if(multi_output_is_allowed_<%=cid%>) {
|
||||
<%=main.getName()%> = new <%=main.getName() %>Struct();
|
||||
}
|
||||
|
||||
<%
|
||||
irToRow.generateConvertRecord("data_" + cid, main.getName(), main.getMetadataTable().getListColumns());
|
||||
}
|
||||
%>
|
||||
} catch (org.talend.components.api.exception.DataRejectException e_<%=cid%>) {
|
||||
java.util.Map<String,Object> info_<%=cid%> = e_<%=cid%>.getRejectInfo();
|
||||
<%
|
||||
if (reject!=null) {
|
||||
%>
|
||||
Object data_<%=cid%> = info_<%=cid%>.get("talend_record");
|
||||
|
||||
if (multi_output_is_allowed_<%=cid%>) {
|
||||
<%=reject.getName()%> = new <%=reject.getName() %>Struct();
|
||||
}
|
||||
try{
|
||||
<%
|
||||
irToRow.generateConvertRecord("data_" + cid, reject.getName());
|
||||
%>
|
||||
}catch(java.lang.Exception e){
|
||||
// do nothing
|
||||
}
|
||||
<%
|
||||
Set<String> commonColumns = new HashSet<String>();
|
||||
|
||||
for (IMetadataColumn column : columnList) {
|
||||
commonColumns.add(column.getLabel());
|
||||
}
|
||||
|
||||
//pass error columns
|
||||
List<IMetadataColumn> rejectColumns = reject.getMetadataTable().getListColumns();
|
||||
for(IMetadataColumn column : rejectColumns) {
|
||||
String columnName = column.getLabel();
|
||||
|
||||
// JavaType javaType = JavaTypesManager.getJavaTypeFromId(column.getTalendType());
|
||||
String typeToGenerate = JavaTypesManager.getTypeToGenerate(column.getTalendType(), column.isNullable());
|
||||
|
||||
//error columns
|
||||
if(!commonColumns.contains(columnName)) {
|
||||
%>
|
||||
<%=reject.getName()%>.<%=columnName%> = (<%=typeToGenerate%>)info_<%=cid%>.get("<%=columnName%>");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
//TODO use a method instead of getting method by the special key "error/errorMessage"
|
||||
Object errorMessage_<%=cid%> = null;
|
||||
if(info_<%=cid%>.containsKey("error")){
|
||||
errorMessage_<%=cid%> = info_<%=cid%>.get("error");
|
||||
}else if(info_<%=cid%>.containsKey("errorMessage")){
|
||||
errorMessage_<%=cid%> = info_<%=cid%>.get("errorMessage");
|
||||
}else{
|
||||
errorMessage_<%=cid%> = "Rejected but error message missing";
|
||||
}
|
||||
errorMessage_<%=cid%> = "Row "+ nb_line_<%=cid %> + ": "+errorMessage_<%=cid%>;
|
||||
System.err.println(errorMessage_<%=cid%>);
|
||||
<%
|
||||
}
|
||||
if (main != null) {
|
||||
%>
|
||||
// If the record is reject, the main line record should put NULL
|
||||
<%=main.getName()%> = null;
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
try {
|
||||
Object data_<%=cid%> = reader_<%=cid%>.getCurrent();
|
||||
<%
|
||||
if (main != null) {
|
||||
%>
|
||||
|
||||
if(multi_output_is_allowed_<%=cid%>) {
|
||||
<%=main.getName()%> = new <%=main.getName() %>Struct();
|
||||
}
|
||||
|
||||
<%
|
||||
irToRow.generateConvertRecord("data_" + cid, main.getName(), main.getMetadataTable().getListColumns());
|
||||
}
|
||||
%>
|
||||
} catch (org.talend.components.api.exception.DataRejectException e_<%=cid%>) {
|
||||
java.util.Map<String,Object> info_<%=cid%> = e_<%=cid%>.getRejectInfo();
|
||||
<%
|
||||
if (reject!=null) {
|
||||
%>
|
||||
Object data_<%=cid%> = info_<%=cid%>.get("talend_record");
|
||||
|
||||
if (multi_output_is_allowed_<%=cid%>) {
|
||||
<%=reject.getName()%> = new <%=reject.getName() %>Struct();
|
||||
}
|
||||
try{
|
||||
<%
|
||||
irToRow.generateConvertRecord("data_" + cid, reject.getName());
|
||||
%>
|
||||
}catch(java.lang.Exception e){
|
||||
// do nothing
|
||||
}
|
||||
<%
|
||||
Set<String> commonColumns = new HashSet<String>();
|
||||
|
||||
for (IMetadataColumn column : columnList) {
|
||||
commonColumns.add(column.getLabel());
|
||||
}
|
||||
|
||||
//pass error columns
|
||||
List<IMetadataColumn> rejectColumns = reject.getMetadataTable().getListColumns();
|
||||
for(IMetadataColumn column : rejectColumns) {
|
||||
String columnName = column.getLabel();
|
||||
|
||||
// JavaType javaType = JavaTypesManager.getJavaTypeFromId(column.getTalendType());
|
||||
String typeToGenerate = JavaTypesManager.getTypeToGenerate(column.getTalendType(), column.isNullable());
|
||||
|
||||
//error columns
|
||||
if(!commonColumns.contains(columnName)) {
|
||||
%>
|
||||
<%=reject.getName()%>.<%=columnName%> = (<%=typeToGenerate%>)info_<%=cid%>.get("<%=columnName%>");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
//TODO use a method instead of getting method by the special key "error/errorMessage"
|
||||
Object errorMessage_<%=cid%> = null;
|
||||
if(info_<%=cid%>.containsKey("error")){
|
||||
errorMessage_<%=cid%> = info_<%=cid%>.get("error");
|
||||
}else if(info_<%=cid%>.containsKey("errorMessage")){
|
||||
errorMessage_<%=cid%> = info_<%=cid%>.get("errorMessage");
|
||||
}else{
|
||||
errorMessage_<%=cid%> = "Rejected but error message missing";
|
||||
}
|
||||
errorMessage_<%=cid%> = "Row "+ nb_line_<%=cid %> + ": "+errorMessage_<%=cid%>;
|
||||
System.err.println(errorMessage_<%=cid%>);
|
||||
<%
|
||||
}
|
||||
if (main != null) {
|
||||
%>
|
||||
// If the record is reject, the main line record should put NULL
|
||||
<%=main.getName()%> = null;
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} // end of catch
|
||||
<%
|
||||
// The for loop around the incoming records from the reader is left open.
|
||||
|
||||
|
||||
} else if (hasInput) {
|
||||
%>
|
||||
org.talend.components.api.component.runtime.Sink sink_<%=cid%> =
|
||||
(org.talend.components.api.component.runtime.Sink)sourceOrSink_<%=cid%>;
|
||||
org.talend.components.api.component.runtime.WriteOperation writeOperation_<%=cid%> = sink_<%=cid%>.createWriteOperation();
|
||||
writeOperation_<%=cid%>.initialize(container_<%=cid%>);
|
||||
org.talend.components.api.component.runtime.Writer writer_<%=cid%> = writeOperation_<%=cid%>.createWriter(container_<%=cid%>);
|
||||
writer_<%=cid%>.open("<%=cid%>");
|
||||
|
||||
resourceMap.put("writer_<%=cid%>", writer_<%=cid%>);
|
||||
org.talend.codegen.enforcer.IncomingSchemaEnforcer incomingEnforcer_<%=cid%> = null;
|
||||
if (sourceOrSink_<%=cid%> instanceof org.talend.components.api.component.runtime.Sink) {
|
||||
org.talend.components.api.component.runtime.Sink sink_<%=cid%> =
|
||||
(org.talend.components.api.component.runtime.Sink)sourceOrSink_<%=cid%>;
|
||||
org.talend.components.api.component.runtime.WriteOperation writeOperation_<%=cid%> = sink_<%=cid%>.createWriteOperation();
|
||||
writeOperation_<%=cid%>.initialize(container_<%=cid%>);
|
||||
writer_<%=cid%> = writeOperation_<%=cid%>.createWriter(container_<%=cid%>);
|
||||
writer_<%=cid%>.open("<%=cid%>");
|
||||
|
||||
resourceMap.put("writer_<%=cid%>", writer_<%=cid%>);
|
||||
} // end of "sourceOrSink_<%=cid%> instanceof ...Sink"
|
||||
org.talend.components.api.component.Connector c_<%=cid%> = null;
|
||||
for (org.talend.components.api.component.Connector currentConnector : props_<%=cid %>.getAvailableConnectors(null, false)) {
|
||||
if (currentConnector.getName().equals("MAIN")) {
|
||||
@@ -405,8 +411,7 @@ if (hasOutputOnly || asInputComponent) {
|
||||
}
|
||||
}
|
||||
org.apache.avro.Schema designSchema_<%=cid%> = props_<%=cid %>.getSchema(c_<%=cid%>, false);
|
||||
org.talend.codegen.enforcer.IncomingSchemaEnforcer incomingEnforcer_<%=cid%>
|
||||
= new org.talend.codegen.enforcer.IncomingSchemaEnforcer(designSchema_<%=cid%>);
|
||||
incomingEnforcer_<%=cid%> = new org.talend.codegen.enforcer.IncomingSchemaEnforcer(designSchema_<%=cid%>);
|
||||
<%
|
||||
List<? extends IConnection> outgoingConns = node.getOutgoingSortedConnections();
|
||||
if (outgoingConns!=null){
|
||||
@@ -442,7 +447,8 @@ if (hasOutputOnly || asInputComponent) {
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
%>
|
||||
|
||||
java.lang.Iterable<?> outgoingMainRecordsList_<%=cid%> = new java.util.ArrayList<Object>();
|
||||
java.util.Iterator outgoingMainRecordsIt_<%=cid%> = null;
|
||||
|
||||
|
||||
@@ -58,13 +58,24 @@ if(isTopologyNone) {
|
||||
else if(hasOutputOnly || asInputComponent){
|
||||
%>
|
||||
} // while
|
||||
reader_<%=cid%>.close();
|
||||
final java.util.Map<String, Object> resultMap_<%=cid%> = reader_<%=cid%>.getReturnValues();
|
||||
<%
|
||||
if (hasOutputOnly || asInputComponent) {
|
||||
%>
|
||||
} // end of "if (sourceOrSink_<%=cid%> instanceof ...Source)"
|
||||
<% } %>
|
||||
java.util.Map<String, Object> resultMap_<%=cid%> = null;
|
||||
if (reader_<%=cid%> != null) {
|
||||
reader_<%=cid%>.close();
|
||||
resultMap_<%=cid%> = reader_<%=cid%>.getReturnValues();
|
||||
}
|
||||
<%
|
||||
}else if(hasInput){
|
||||
%>
|
||||
org.talend.components.api.component.runtime.Result resultObject_<%=cid%> = (org.talend.components.api.component.runtime.Result)writer_<%=cid%>.close();
|
||||
final java.util.Map<String, Object> resultMap_<%=cid%> = writer_<%=cid%>.getWriteOperation().finalize(java.util.Arrays.<org.talend.components.api.component.runtime.Result>asList(resultObject_<%=cid%>), container_<%=cid%>);
|
||||
java.util.Map<String, Object> resultMap_<%=cid%> = null;
|
||||
if (writer_<%=cid%> != null) {
|
||||
org.talend.components.api.component.runtime.Result resultObject_<%=cid%> = (org.talend.components.api.component.runtime.Result)writer_<%=cid%>.close();
|
||||
resultMap_<%=cid%> = writer_<%=cid%>.getWriteOperation().finalize(java.util.Arrays.<org.talend.components.api.component.runtime.Result>asList(resultObject_<%=cid%>), container_<%=cid%>);
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
return stringBuffer.toString();
|
||||
|
||||
@@ -84,7 +84,7 @@ if(hasInput){
|
||||
for (int i = 0; i < input_columnList.size(); i++) {
|
||||
if(!input_columnList.get(i).getTalendType().equals("id_Dynamic")) {
|
||||
%>
|
||||
if (incomingEnforcer_<%=cid%>.getDesignSchema().getField("<%=input_columnList.get(i)%>") == null){
|
||||
if (incomingEnforcer_<%=cid%> != null && incomingEnforcer_<%=cid%>.getDesignSchema().getField("<%=input_columnList.get(i)%>") == null){
|
||||
incomingEnforcer_<%=cid%>.addIncomingNodeField("<%=input_columnList.get(i)%>", ((Object) <%=inputConn.getName()%>.<%=input_columnList.get(i)%>).getClass().getCanonicalName());
|
||||
shouldCreateRuntimeSchemaForIncomingNode = true;
|
||||
}
|
||||
@@ -92,7 +92,7 @@ if(hasInput){
|
||||
}
|
||||
}
|
||||
%>
|
||||
if (shouldCreateRuntimeSchemaForIncomingNode){
|
||||
if (shouldCreateRuntimeSchemaForIncomingNode && incomingEnforcer_<%=cid%> != null){
|
||||
incomingEnforcer_<%=cid%>.createRuntimeSchema();
|
||||
}
|
||||
<%
|
||||
@@ -111,7 +111,7 @@ if(hasInput){
|
||||
|
||||
if (dynamicPos != -1) {
|
||||
%>
|
||||
if (!incomingEnforcer_<%=cid%>.areDynamicFieldsInitialized()) {
|
||||
if (incomingEnforcer_<%=cid%> != null && !incomingEnforcer_<%=cid%>.areDynamicFieldsInitialized()) {
|
||||
// Initialize the dynamic columns when they are first encountered.
|
||||
for (routines.system.DynamicMetadata dm_<%=cid%> : <%=inputConn.getName()%>.<%=input_columnList.get(dynamicPos).getLabel()%>.metadatas) {
|
||||
incomingEnforcer_<%=cid%>.addDynamicField(
|
||||
@@ -128,22 +128,26 @@ if(hasInput){
|
||||
}
|
||||
|
||||
%>
|
||||
incomingEnforcer_<%=cid%>.createNewRecord();
|
||||
if (incomingEnforcer_<%=cid%> != null) {
|
||||
incomingEnforcer_<%=cid%>.createNewRecord();
|
||||
}
|
||||
<%
|
||||
for (int i = 0; i < input_columnList.size(); i++) { // column
|
||||
IMetadataColumn column = input_columnList.get(i);
|
||||
if (dynamicPos != i) {
|
||||
%>
|
||||
//skip the put action if the input column doesn't appear in component runtime schema
|
||||
if (incomingEnforcer_<%=cid%>.getRuntimeSchema().getField("<%=input_columnList.get(i)%>") != null){
|
||||
if (incomingEnforcer_<%=cid%> != null && incomingEnforcer_<%=cid%>.getRuntimeSchema().getField("<%=input_columnList.get(i)%>") != null){
|
||||
incomingEnforcer_<%=cid%>.put("<%=column.getLabel()%>", <%=inputConn.getName()%>.<%=column.getLabel()%>);
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
for (int i = 0; i < <%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnCount(); i++) {
|
||||
incomingEnforcer_<%=cid%>.put(<%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnMetadata(i).getName(),
|
||||
<%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnValue(i));
|
||||
if (incomingEnforcer_<%=cid%> != null) {
|
||||
for (int i = 0; i < <%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnCount(); i++) {
|
||||
incomingEnforcer_<%=cid%>.put(<%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnMetadata(i).getName(),
|
||||
<%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnValue(i));
|
||||
}
|
||||
}
|
||||
<%
|
||||
}
|
||||
@@ -177,7 +181,11 @@ if(hasInput){
|
||||
} // propInfo
|
||||
|
||||
%>
|
||||
org.apache.avro.generic.IndexedRecord data_<%=cid%> = incomingEnforcer_<%=cid%>.getCurrentRecord();
|
||||
|
||||
org.apache.avro.generic.IndexedRecord data_<%=cid%> = null;
|
||||
if (incomingEnforcer_<%=cid%> != null) {
|
||||
data_<%=cid%> = incomingEnforcer_<%=cid%>.getCurrentRecord();
|
||||
}
|
||||
|
||||
<%
|
||||
boolean isParallelize ="true".equalsIgnoreCase(ElementParameterParser.getValue(node, "__PARALLELIZE__"));
|
||||
@@ -190,8 +198,9 @@ if(hasInput){
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
writer_<%=cid%>.write(data_<%=cid%>);
|
||||
if (writer_<%=cid%> != null && data_<%=cid%> != null) {
|
||||
writer_<%=cid%>.write(data_<%=cid%>);
|
||||
}
|
||||
|
||||
nb_line_<%=cid %>++;
|
||||
<%if(hasMainOutput){
|
||||
|
||||
@@ -437,6 +437,20 @@ private RunTrace runTrace = new RunTrace();
|
||||
globalMap.put(KEY_DB_DATASOURCES, talendDataSources);
|
||||
globalMap.put(KEY_DB_DATASOURCES_RAW, new java.util.HashMap<String, javax.sql.DataSource>(dataSources));
|
||||
}
|
||||
|
||||
public void setDataSourceReferences(List serviceReferences) throws Exception{
|
||||
|
||||
java.util.Map<String, routines.system.TalendDataSource> talendDataSources = new java.util.HashMap<String, routines.system.TalendDataSource>();
|
||||
java.util.Map<String, javax.sql.DataSource> dataSources = new java.util.HashMap<String, javax.sql.DataSource>();
|
||||
|
||||
for (java.util.Map.Entry<String, javax.sql.DataSource> entry : BundleUtils.getServices(serviceReferences, javax.sql.DataSource.class).entrySet()) {
|
||||
dataSources.put(entry.getKey(), entry.getValue());
|
||||
talendDataSources.put(entry.getKey(), new routines.system.TalendDataSource(entry.getValue()));
|
||||
}
|
||||
|
||||
globalMap.put(KEY_DB_DATASOURCES, talendDataSources);
|
||||
globalMap.put(KEY_DB_DATASOURCES_RAW, new java.util.HashMap<String, javax.sql.DataSource>(dataSources));
|
||||
}
|
||||
|
||||
<%
|
||||
for (INode logCatcher : process.getNodesOfType("tLogCatcher")) {
|
||||
|
||||
@@ -44,7 +44,7 @@
|
||||
INode startNode = subTree.getRootNode();
|
||||
String startNodeId = startNode.getUniqueName();
|
||||
|
||||
if(startNodeId!=null && startNodeId.startsWith("tCollector")) {
|
||||
if ("tCollector".equals( startNode.getComponent().getOriginalName() )) {
|
||||
List<? extends INode> departitioners = startNode.getProcess().getNodesOfType("tDepartitioner");
|
||||
if(departitioners!=null) {
|
||||
for(INode departitioner : departitioners) {
|
||||
|
||||
@@ -68,6 +68,14 @@
|
||||
id="org.talend.designer.components.model.UserComponentsProvider">
|
||||
</ComponentsProvider>
|
||||
</extension>
|
||||
<extension
|
||||
point="org.talend.core.components_provider">
|
||||
<ComponentsProvider
|
||||
class="org.talend.designer.codegen.components.model.SharedStudioUserComponentProvider"
|
||||
folderName="user"
|
||||
id="org.talend.designer.codegen.components.model.SharedStudioUserComponentProvider">
|
||||
</ComponentsProvider>
|
||||
</extension>
|
||||
<extension
|
||||
point="org.eclipse.core.runtime.preferences">
|
||||
<initializer
|
||||
|
||||
@@ -26,10 +26,8 @@ import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.ResourceBundle;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
@@ -75,6 +73,7 @@ import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.model.components.IComponentsHandler;
|
||||
import org.talend.core.model.components.filters.ComponentsFactoryProviderManager;
|
||||
import org.talend.core.model.components.filters.IComponentFactoryFilter;
|
||||
import org.talend.core.runtime.util.ComponentsLocationProvider;
|
||||
import org.talend.core.ui.IJobletProviderService;
|
||||
import org.talend.core.ui.ISparkJobletProviderService;
|
||||
import org.talend.core.ui.ISparkStreamingJobletProviderService;
|
||||
@@ -83,8 +82,6 @@ import org.talend.core.ui.images.CoreImageProvider;
|
||||
import org.talend.core.utils.TalendCacheUtils;
|
||||
import org.talend.designer.codegen.CodeGeneratorActivator;
|
||||
import org.talend.designer.codegen.i18n.Messages;
|
||||
import org.talend.designer.core.ITisLocalProviderService;
|
||||
import org.talend.designer.core.ITisLocalProviderService.ResClassLoader;
|
||||
import org.talend.designer.core.model.components.ComponentBundleToPath;
|
||||
import org.talend.designer.core.model.components.ComponentFilesNaming;
|
||||
import org.talend.designer.core.model.components.EmfComponent;
|
||||
@@ -164,7 +161,11 @@ public class ComponentsFactory implements IComponentsFactory {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
isInitialising.set(true);
|
||||
removeOldComponentsUserFolder(); // not used anymore
|
||||
try {
|
||||
removeOldComponentsUserFolder();
|
||||
} catch (IOException ex) {
|
||||
ExceptionHandler.process(ex);
|
||||
} // not used anymore
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// TimeMeasure.display = true;
|
||||
@@ -387,10 +388,12 @@ public class ComponentsFactory implements IComponentsFactory {
|
||||
ComponentManager.saveResource();
|
||||
}
|
||||
|
||||
private void removeOldComponentsUserFolder() {
|
||||
private void removeOldComponentsUserFolder() throws IOException {
|
||||
String userPath = IComponentsFactory.COMPONENTS_INNER_FOLDER + File.separatorChar
|
||||
+ ComponentUtilities.getExtFolder(OLD_COMPONENTS_USER_INNER_FOLDER);
|
||||
File componentsLocation = getComponentsLocation(userPath);
|
||||
ComponentsProviderManager componentsProviderManager = ComponentsProviderManager.getInstance();
|
||||
AbstractComponentsProvider componentsProvider = componentsProviderManager.loadUserComponentsProvidersFromExtension();
|
||||
File componentsLocation = getComponentsLocation(componentsProvider, userPath);
|
||||
if (componentsLocation != null && componentsLocation.exists()) {
|
||||
FilesUtils.removeFolder(componentsLocation, true);
|
||||
}
|
||||
@@ -671,114 +674,38 @@ public class ComponentsFactory implements IComponentsFactory {
|
||||
*
|
||||
* @param currentFolder
|
||||
* @return
|
||||
* @throws IOException
|
||||
* @throws BusinessException
|
||||
*/
|
||||
|
||||
private File getComponentsLocation(String folder) {
|
||||
String componentsPath = IComponentsFactory.COMPONENTS_LOCATION;
|
||||
IBrandingService breaningService = (IBrandingService) GlobalServiceRegister.getDefault()
|
||||
.getService(IBrandingService.class);
|
||||
if (breaningService.isPoweredOnlyCamel()) {
|
||||
componentsPath = IComponentsFactory.CAMEL_COMPONENTS_LOCATION;
|
||||
}
|
||||
Bundle b = Platform.getBundle(componentsPath);
|
||||
|
||||
File file = null;
|
||||
try {
|
||||
URL url = FileLocator.find(b, new Path(folder), null);
|
||||
if (url == null) {
|
||||
return null;
|
||||
private File getComponentsLocation(AbstractComponentsProvider componentsProvider, String folder) throws IOException {
|
||||
|
||||
if (componentsProvider instanceof ComponentsLocationProvider) {
|
||||
return componentsProvider.getInstallationFolder();
|
||||
} else {
|
||||
String componentsPath = IComponentsFactory.COMPONENTS_LOCATION;
|
||||
IBrandingService breaningService = (IBrandingService) GlobalServiceRegister.getDefault()
|
||||
.getService(IBrandingService.class);
|
||||
if (breaningService.isPoweredOnlyCamel()) {
|
||||
componentsPath = IComponentsFactory.CAMEL_COMPONENTS_LOCATION;
|
||||
}
|
||||
URL fileUrl = FileLocator.toFileURL(url);
|
||||
file = new File(fileUrl.getPath());
|
||||
} catch (Exception e) {
|
||||
// e.printStackTrace();
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
Bundle b = Platform.getBundle(componentsPath);
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
private File getComponentsLocation(String folder, AbstractComponentsProvider provider) {
|
||||
File file = null;
|
||||
try {
|
||||
if (provider != null) {
|
||||
file = provider.getInstallationFolder();
|
||||
} else {
|
||||
String componentsPath = IComponentsFactory.COMPONENTS_LOCATION;
|
||||
Bundle b = Platform.getBundle(componentsPath);
|
||||
IBrandingService breaningService = (IBrandingService) GlobalServiceRegister.getDefault()
|
||||
.getService(IBrandingService.class);
|
||||
if (breaningService.isPoweredOnlyCamel()) {
|
||||
componentsPath = IComponentsFactory.CAMEL_COMPONENTS_LOCATION;
|
||||
}
|
||||
File file = null;
|
||||
try {
|
||||
URL url = FileLocator.find(b, new Path(folder), null);
|
||||
if (url == null) {
|
||||
return null;
|
||||
}
|
||||
URL fileUrl = FileLocator.toFileURL(url);
|
||||
file = new File(fileUrl.getPath());
|
||||
|
||||
} catch (Exception e) {
|
||||
// e.printStackTrace();
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private ResourceBundle getComponentResourceBundle(IComponent currentComp, String source, String cachedPathSource,
|
||||
AbstractComponentsProvider provider) {
|
||||
try {
|
||||
AbstractComponentsProvider currentProvider = provider;
|
||||
if (currentProvider == null) {
|
||||
ComponentsProviderManager componentsProviderManager = ComponentsProviderManager.getInstance();
|
||||
Collection<AbstractComponentsProvider> providers = componentsProviderManager.getProviders();
|
||||
for (AbstractComponentsProvider curProvider : providers) {
|
||||
String path = new Path(curProvider.getInstallationFolder().toString()).toPortableString();
|
||||
if (source.startsWith(path)) {
|
||||
// fix for TDI-19889 and TDI-20507 to get the correct component provider
|
||||
if (cachedPathSource != null) {
|
||||
if (path.contains(cachedPathSource)) {
|
||||
currentProvider = curProvider;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
currentProvider = curProvider;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
String installPath = currentProvider.getInstallationFolder().toString();
|
||||
String label = ComponentFilesNaming.getInstance().getBundleName(currentComp.getName(),
|
||||
installPath.substring(installPath.lastIndexOf(IComponentsFactory.COMPONENTS_INNER_FOLDER)));
|
||||
|
||||
if (currentProvider.isUseLocalProvider()) {
|
||||
// if the component use local provider as storage (for user / ecosystem components)
|
||||
// then get the bundle resource from the current main component provider.
|
||||
|
||||
// note: code here to review later, service like this shouldn't be used...
|
||||
ResourceBundle bundle = null;
|
||||
IBrandingService brandingService = (IBrandingService) GlobalServiceRegister.getDefault()
|
||||
.getService(IBrandingService.class);
|
||||
if (brandingService.isPoweredOnlyCamel()) {
|
||||
bundle = currentProvider.getResourceBundle(label);
|
||||
} else {
|
||||
ITisLocalProviderService service = (ITisLocalProviderService) GlobalServiceRegister.getDefault()
|
||||
.getService(ITisLocalProviderService.class);
|
||||
bundle = service.getResourceBundle(label);
|
||||
}
|
||||
return bundle;
|
||||
} else {
|
||||
ResourceBundle bundle = ResourceBundle.getBundle(label, Locale.getDefault(),
|
||||
new ResClassLoader(currentProvider.getClass().getClassLoader()));
|
||||
return bundle;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
|
||||
return null;
|
||||
return file;
|
||||
}
|
||||
}
|
||||
|
||||
private String getCodeLanguageSuffix() {
|
||||
@@ -1082,5 +1009,13 @@ public class ComponentsFactory implements IComponentsFactory {
|
||||
public void setComponentsHandler(IComponentsHandler componentsHandler) {
|
||||
this.componentsHandler = componentsHandler;
|
||||
}
|
||||
|
||||
public String getCustomComponentBundlePath() {
|
||||
ComponentsProviderManager componentsProviderManager = ComponentsProviderManager.getInstance();
|
||||
AbstractComponentsProvider componentsProvider = componentsProviderManager.loadUserComponentsProvidersFromExtension();
|
||||
String bundle = componentsProvider.getComponentsBundle();
|
||||
return ComponentBundleToPath.getPathFromBundle(bundle);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import org.eclipse.core.runtime.IExtensionRegistry;
|
||||
import org.eclipse.core.runtime.Platform;
|
||||
import org.talend.core.GlobalServiceRegister;
|
||||
import org.talend.core.model.components.AbstractComponentsProvider;
|
||||
import org.talend.core.runtime.util.SharedStudioInfoProvider;
|
||||
import org.talend.core.ui.branding.IBrandingService;
|
||||
import org.talend.designer.codegen.i18n.Messages;
|
||||
|
||||
@@ -69,6 +70,9 @@ public final class ComponentsProviderManager {
|
||||
try {
|
||||
AbstractComponentsProvider componentsProvider = (AbstractComponentsProvider) configurationElement
|
||||
.createExecutableExtension("class"); //$NON-NLS-1$
|
||||
if (componentsProvider instanceof SharedStudioInfoProvider && !((SharedStudioInfoProvider)componentsProvider).isSupportCurrentMode()) {
|
||||
continue;
|
||||
}
|
||||
componentsProvider.setId(id);
|
||||
componentsProvider.setFolderName(folderName);
|
||||
componentsProvider.setContributer(contributerName);
|
||||
@@ -81,15 +85,15 @@ public final class ComponentsProviderManager {
|
||||
}
|
||||
}
|
||||
|
||||
public AbstractComponentsProvider loadUserComponentsProvidersFromExtension() {
|
||||
if (providers == null) {
|
||||
loadComponentsProvidersFromExtension();
|
||||
}
|
||||
for (AbstractComponentsProvider provider : providers) {
|
||||
if ("org.talend.designer.components.model.UserComponentsProvider".equals(provider.getId())) {
|
||||
return provider;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
public AbstractComponentsProvider loadUserComponentsProvidersFromExtension() {
|
||||
if (providers == null) {
|
||||
loadComponentsProvidersFromExtension();
|
||||
}
|
||||
for (AbstractComponentsProvider provider : providers) {
|
||||
if (provider instanceof UserComponentsProvider) {
|
||||
return provider;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package org.talend.designer.codegen.components.model;
|
||||
//============================================================================
|
||||
//
|
||||
//Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
//
|
||||
//This source code is available under agreement available at
|
||||
//%InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
//
|
||||
//You should have received a copy of the agreement
|
||||
//along with this program; if not, write to Talend SA
|
||||
//9 rue Pages 92150 Suresnes, France
|
||||
//
|
||||
//============================================================================
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.util.ResourceBundle;
|
||||
|
||||
import org.eclipse.core.runtime.IPath;
|
||||
import org.eclipse.core.runtime.Path;
|
||||
import org.eclipse.core.runtime.Platform;
|
||||
import org.talend.core.model.components.ComponentUtilities;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.runtime.util.ComponentsLocationProvider;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.designer.core.model.components.ComponentBundleToPath;
|
||||
|
||||
public class SharedStudioUserComponentProvider extends UserComponentsProvider implements ComponentsLocationProvider{
|
||||
@Override
|
||||
public File getInstallationFolder() throws IOException {
|
||||
File componentFolder = SharedStudioUtils.getSharedStudioComponentsParentFolder();
|
||||
IPath path = new Path(IComponentsFactory.COMPONENTS_INNER_FOLDER);
|
||||
path = path.append(IComponentsFactory.EXTERNAL_COMPONENTS_INNER_FOLDER).append(ComponentUtilities.getExtFolder(getFolderName()));
|
||||
File installationFolder = new File (componentFolder, path.toOSString());
|
||||
return installationFolder;
|
||||
}
|
||||
|
||||
|
||||
public String getComponentsBundle() {
|
||||
return ComponentBundleToPath.SHARED_STUDIO_CUSTOM_COMPONENT_BUNDLE;
|
||||
}
|
||||
|
||||
public boolean isSupportCurrentMode() {
|
||||
if (SharedStudioUtils.isSharedStudioMode()) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResourceBundle getResourceBundle(String label) {
|
||||
URL configFolderUrl = Platform.getConfigurationLocation().getURL();
|
||||
URLClassLoader urlLoader = new URLClassLoader(new java.net.URL[]{configFolderUrl});
|
||||
java.util.ResourceBundle bundle = java.util.ResourceBundle.getBundle( label ,
|
||||
java.util.Locale.getDefault(), urlLoader );
|
||||
return bundle;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -34,13 +34,15 @@ import org.talend.core.model.components.ComponentUtilities;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.model.general.Project;
|
||||
import org.talend.core.model.repository.ERepositoryObjectType;
|
||||
import org.talend.core.runtime.util.SharedStudioInfoProvider;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.core.ui.branding.IBrandingService;
|
||||
import org.talend.designer.codegen.CodeGeneratorActivator;
|
||||
import org.talend.designer.codegen.components.ui.IComponentPreferenceConstant;
|
||||
import org.talend.repository.ProjectManager;
|
||||
|
||||
/***/
|
||||
public class UserComponentsProvider extends AbstractCustomComponentsProvider {
|
||||
public class UserComponentsProvider extends AbstractCustomComponentsProvider implements SharedStudioInfoProvider{
|
||||
|
||||
@Override
|
||||
protected File getExternalComponentsLocation() {
|
||||
@@ -147,5 +149,11 @@ public class UserComponentsProvider extends AbstractCustomComponentsProvider {
|
||||
public String getComponentsBundle() {
|
||||
return IComponentsFactory.COMPONENTS_LOCATION;
|
||||
}
|
||||
|
||||
|
||||
public boolean isSupportCurrentMode() {
|
||||
if (SharedStudioUtils.isSharedStudioMode()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import java.util.Map;
|
||||
import org.eclipse.core.runtime.Platform;
|
||||
import org.talend.commons.exception.ExceptionHandler;
|
||||
import org.talend.commons.utils.StringUtils;
|
||||
import org.talend.designer.core.model.components.ComponentBundleToPath;
|
||||
|
||||
/**
|
||||
* Jet container for a particular component.
|
||||
@@ -213,8 +214,17 @@ public class JetBean {
|
||||
if (pluginIdToBundle.containsKey(pluginId)) {
|
||||
base = pluginIdToBundle.get(pluginId);
|
||||
} else {
|
||||
base = Platform.getBundle(pluginId).getEntry("/").toString(); //$NON-NLS-1$
|
||||
pluginIdToBundle.put(pluginId, base);
|
||||
if (ComponentBundleToPath.SHARED_STUDIO_CUSTOM_COMPONENT_BUNDLE.equals(pluginId)) {
|
||||
base = ComponentBundleToPath.getPathFromBundle(pluginId);
|
||||
if (!base.endsWith("/")) {
|
||||
base = base + "/";
|
||||
}
|
||||
pluginIdToBundle.put(pluginId, base);
|
||||
} else {
|
||||
base = Platform.getBundle(pluginId).getEntry("/").toString(); //$NON-NLS-1$
|
||||
pluginIdToBundle.put(pluginId, base);
|
||||
}
|
||||
|
||||
}
|
||||
String result = base + relativeUri;
|
||||
return result;
|
||||
|
||||
@@ -136,13 +136,11 @@ public class TalendJETCompiler extends JETCompiler {
|
||||
// get the plugin name from fileURI
|
||||
String refPluginName = matcher.group(1);
|
||||
// retrieve the plugin URI by pluginName.
|
||||
Bundle refBundle = Platform.getBundle(refPluginName);
|
||||
if (refBundle != null) {
|
||||
String realURI = TemplateUtil.getPlatformUrlOfBundle(refPluginName);
|
||||
String realURI = TemplateUtil.getPlatformUrlOfBundle(refPluginName);
|
||||
if (realURI != null) {
|
||||
// replace the old fileURI to new one by pluginURI
|
||||
String newFileURI = fileURI.replaceFirst(PLUGIN_VAR_PATTERN.pattern(), realURI);
|
||||
return newFileURI;
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ package org.talend.designer.codegen.config;
|
||||
|
||||
import org.eclipse.core.runtime.Platform;
|
||||
import org.osgi.framework.Bundle;
|
||||
import org.talend.designer.core.model.components.ComponentBundleToPath;
|
||||
|
||||
/**
|
||||
* CodeGenerator Templates Ressources Utils.
|
||||
@@ -161,10 +162,25 @@ public class TemplateUtil {
|
||||
* @return
|
||||
*/
|
||||
public static String getPlatformUrlOfBundle(String bundleName) {
|
||||
Bundle bundle = Platform.getBundle(bundleName);
|
||||
if (bundle == null) {
|
||||
return null;
|
||||
}
|
||||
return "platform:/plugin/" + bundle.getSymbolicName() + "_" + bundle.getVersion().toString() + "/";
|
||||
if (ComponentBundleToPath.SHARED_STUDIO_CUSTOM_COMPONENT_BUNDLE.equals(bundleName)) {
|
||||
String basePath = ComponentBundleToPath.getPathFromBundle(bundleName);
|
||||
if (!basePath.endsWith("/")) {
|
||||
basePath = basePath + "/";
|
||||
}
|
||||
return basePath;
|
||||
} else {
|
||||
Bundle bundle = Platform.getBundle(bundleName);
|
||||
if (bundle == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("platform:/plugin/");
|
||||
sb.append(bundle.getSymbolicName());
|
||||
sb.append("_");
|
||||
sb.append(bundle.getVersion().toString());
|
||||
sb.append("/");
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,6 +47,7 @@ import org.talend.core.ui.component.ComponentsFactoryProvider;
|
||||
import org.talend.designer.codegen.CodeGeneratorActivator;
|
||||
import org.talend.designer.codegen.config.TemplateUtil;
|
||||
import org.talend.designer.codegen.i18n.Messages;
|
||||
import org.talend.designer.core.model.components.ComponentBundleToPath;
|
||||
|
||||
/**
|
||||
* DOC xtan
|
||||
@@ -256,10 +257,9 @@ public final class JetSkeletonManager {
|
||||
};
|
||||
|
||||
for (TemplateUtil template : CodeGeneratorInternalTemplatesFactoryProvider.getInstance().getTemplates()) {
|
||||
Bundle b = Platform.getBundle(template.getJetPluginRepository());
|
||||
URL resourcesUrl = null;
|
||||
try {
|
||||
resourcesUrl = FileLocator.toFileURL(FileLocator.find(b, new Path(template.getTemplateRelativeUri()), null));
|
||||
resourcesUrl = FileLocator.toFileURL(ComponentBundleToPath.findComponentsBundleURL(template.getJetPluginRepository(), new Path(template.getTemplateRelativeUri()), null));
|
||||
} catch (IOException e) {
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
|
||||
@@ -9,6 +9,14 @@
|
||||
id="org.talend.designer.components.exchange.ExchangeComponentsProvider">
|
||||
</ComponentsProvider>
|
||||
</extension>
|
||||
<extension
|
||||
point="org.talend.core.components_provider">
|
||||
<ComponentsProvider
|
||||
class="org.talend.designer.components.exchange.SharedStudioExchangeComponentsProvider"
|
||||
folderName="exchange"
|
||||
id="org.talend.designer.components.exchange.SharedStudioExchangeComponentsProvider">
|
||||
</ComponentsProvider>
|
||||
</extension>
|
||||
<extension
|
||||
point="org.talend.core.runtime.service">
|
||||
<Service
|
||||
|
||||
@@ -28,13 +28,15 @@ import org.talend.core.GlobalServiceRegister;
|
||||
import org.talend.core.model.components.AbstractComponentsProvider;
|
||||
import org.talend.core.model.components.ComponentUtilities;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.runtime.util.SharedStudioInfoProvider;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.core.ui.branding.IBrandingService;
|
||||
import org.talend.designer.components.exchange.util.ExchangeUtils;
|
||||
|
||||
/**
|
||||
* DOC hcyi class global comment. Detailled comment
|
||||
*/
|
||||
public class ExchangeComponentsProvider extends AbstractComponentsProvider {
|
||||
public class ExchangeComponentsProvider extends AbstractComponentsProvider implements SharedStudioInfoProvider{
|
||||
|
||||
/**
|
||||
* ExchangeComponentsProvider constructor.
|
||||
@@ -184,4 +186,10 @@ public class ExchangeComponentsProvider extends AbstractComponentsProvider {
|
||||
return IComponentsFactory.COMPONENTS_LOCATION;
|
||||
}
|
||||
|
||||
public boolean isSupportCurrentMode() {
|
||||
if (SharedStudioUtils.isSharedStudioMode()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package org.talend.designer.components.exchange;
|
||||
//============================================================================
|
||||
//
|
||||
//Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
//
|
||||
//This source code is available under agreement available at
|
||||
//%InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
//
|
||||
//You should have received a copy of the agreement
|
||||
//along with this program; if not, write to Talend SA
|
||||
//9 rue Pages 92150 Suresnes, France
|
||||
//
|
||||
//============================================================================
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.util.ResourceBundle;
|
||||
|
||||
import org.eclipse.core.runtime.IPath;
|
||||
import org.eclipse.core.runtime.Path;
|
||||
import org.eclipse.core.runtime.Platform;
|
||||
import org.talend.core.model.components.ComponentUtilities;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.runtime.util.ComponentsLocationProvider;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.designer.core.model.components.ComponentBundleToPath;
|
||||
|
||||
public class SharedStudioExchangeComponentsProvider extends ExchangeComponentsProvider implements ComponentsLocationProvider{
|
||||
@Override
|
||||
public File getInstallationFolder() throws IOException {
|
||||
File componentFolder = SharedStudioUtils.getSharedStudioComponentsParentFolder();
|
||||
IPath path = new Path(IComponentsFactory.COMPONENTS_INNER_FOLDER);
|
||||
path = path.append(IComponentsFactory.EXTERNAL_COMPONENTS_INNER_FOLDER).append(ComponentUtilities.getExtFolder(getFolderName()));
|
||||
File installationFolder = new File (componentFolder, path.toOSString());
|
||||
return installationFolder;
|
||||
}
|
||||
|
||||
|
||||
public String getComponentsBundle() {
|
||||
return ComponentBundleToPath.SHARED_STUDIO_CUSTOM_COMPONENT_BUNDLE;
|
||||
}
|
||||
|
||||
public boolean isSupportCurrentMode() {
|
||||
if (SharedStudioUtils.isSharedStudioMode()) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResourceBundle getResourceBundle(String label) {
|
||||
URL configFolderUrl = Platform.getConfigurationLocation().getURL();
|
||||
URLClassLoader urlLoader = new URLClassLoader(new java.net.URL[]{configFolderUrl});
|
||||
java.util.ResourceBundle bundle = java.util.ResourceBundle.getBundle( label ,
|
||||
java.util.Locale.getDefault(), urlLoader );
|
||||
return bundle;
|
||||
}
|
||||
}
|
||||
@@ -52,6 +52,7 @@ import org.talend.core.download.IDownloadHelper;
|
||||
import org.talend.core.model.components.ComponentManager;
|
||||
import org.talend.core.model.components.IComponent;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.core.ui.component.ComponentPaletteUtilities;
|
||||
import org.talend.core.ui.component.ComponentsFactoryProvider;
|
||||
import org.talend.designer.codegen.ICodeGeneratorService;
|
||||
@@ -312,51 +313,54 @@ public class DownloadComponenentsAction extends Action implements IIntroAction {
|
||||
|
||||
protected void afterDownload(IProgressMonitor monitor, ComponentExtension extension, File localZipFile) throws Exception {
|
||||
if (UpdatesHelper.isComponentUpdateSite(localZipFile)) {
|
||||
final File workFolder = org.talend.utils.files.FileUtils.createTmpFolder("downloadedComponents", ""); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
if (!SharedStudioUtils.isSharedStudioMode()) {
|
||||
final File workFolder = org.talend.utils.files.FileUtils.createTmpFolder("downloadedComponents", ""); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
|
||||
try {
|
||||
FilesUtils.copyFile(localZipFile, new File(workFolder, localZipFile.getName()));
|
||||
try {
|
||||
FilesUtils.copyFile(localZipFile, new File(workFolder, localZipFile.getName()));
|
||||
|
||||
ComponentsInstallComponent component = LocalComponentInstallHelper.getComponent();
|
||||
if (component != null) {
|
||||
try {
|
||||
component.setComponentFolder(workFolder);
|
||||
if (component.install()) {
|
||||
ComponentsInstallComponent component = LocalComponentInstallHelper.getComponent();
|
||||
if (component != null) {
|
||||
try {
|
||||
component.setComponentFolder(workFolder);
|
||||
if (component.install()) {
|
||||
|
||||
if (component.needRelaunch()) {
|
||||
askReboot();
|
||||
} else {
|
||||
MessageDialog.openInformation(DisplayUtils.getDefaultShell(),
|
||||
Messages.getString("DownloadComponenentsAction.installComponentsTitle"),
|
||||
component.getInstalledMessages());
|
||||
|
||||
if (component.needRelaunch()) {
|
||||
askReboot();
|
||||
} else {
|
||||
MessageDialog.openInformation(DisplayUtils.getDefaultShell(),
|
||||
Messages.getString("DownloadComponenentsAction.installComponentsTitle"),
|
||||
component.getInstalledMessages());
|
||||
}
|
||||
} else {// install failure
|
||||
MessageDialog.openWarning(DisplayUtils.getDefaultShell(),
|
||||
Messages.getString("DownloadComponenentsAction_failureTitle"), //$NON-NLS-1$
|
||||
Messages.getString("DownloadComponenentsAction_failureMessage", extension.getLabel())); //$NON-NLS-1$
|
||||
}
|
||||
} else {// install failure
|
||||
MessageDialog.openWarning(DisplayUtils.getDefaultShell(),
|
||||
} finally {
|
||||
// after install, clear the setting for service.
|
||||
component.setComponentFolder(null);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// Popup dialog to user to waring install failed.
|
||||
Display.getDefault().syncExec(new Runnable() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
MessageDialog.openError(DisplayUtils.getDefaultShell(false),
|
||||
Messages.getString("DownloadComponenentsAction_failureTitle"), //$NON-NLS-1$
|
||||
Messages.getString("DownloadComponenentsAction_failureMessage", extension.getLabel())); //$NON-NLS-1$
|
||||
}
|
||||
} finally {
|
||||
// after install, clear the setting for service.
|
||||
component.setComponentFolder(null);
|
||||
}
|
||||
});
|
||||
throw e;
|
||||
} finally {
|
||||
FilesUtils.deleteFolder(workFolder, true);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// Popup dialog to user to waring install failed.
|
||||
Display.getDefault().syncExec(new Runnable() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
MessageDialog.openError(DisplayUtils.getDefaultShell(false),
|
||||
Messages.getString("DownloadComponenentsAction_failureTitle"), //$NON-NLS-1$
|
||||
Messages.getString("DownloadComponenentsAction_failureMessage", extension.getLabel())); //$NON-NLS-1$
|
||||
}
|
||||
});
|
||||
throw e;
|
||||
} finally {
|
||||
FilesUtils.deleteFolder(workFolder, true);
|
||||
}
|
||||
monitor.done();
|
||||
ExchangeManager.getInstance().saveDownloadedExtensionsToFile(extension);
|
||||
monitor.done();
|
||||
ExchangeManager.getInstance().saveDownloadedExtensionsToFile(extension);
|
||||
}
|
||||
} else {
|
||||
File installedLocation = ComponentInstaller.unzip(localZipFile.getAbsolutePath(), getComponentsFolder()
|
||||
.getAbsolutePath());
|
||||
|
||||
@@ -37,6 +37,7 @@ import org.eclipse.swt.widgets.Shell;
|
||||
import org.eclipse.ui.PlatformUI;
|
||||
import org.talend.commons.ui.runtime.exception.ExceptionHandler;
|
||||
import org.talend.core.download.DownloadHelper;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.designer.components.exchange.i18n.Messages;
|
||||
import org.talend.designer.components.exchange.model.Category;
|
||||
import org.talend.designer.components.exchange.model.VersionRevision;
|
||||
@@ -105,7 +106,7 @@ public class ImportExchangeDialog extends Dialog {
|
||||
|
||||
@Override
|
||||
protected void okPressed() {
|
||||
IPath tempPath = new Path(System.getProperty("user.dir")).append("temp"); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
IPath tempPath = SharedStudioUtils.getTempFolderPath();
|
||||
File pathFile = tempPath.toFile();
|
||||
if (downloadproperty.getFileName() == null || downloadproperty.getFileName() == null) {
|
||||
MessageBox box = new MessageBox(Display.getCurrent().getActiveShell(), SWT.ICON_WARNING | SWT.OK);
|
||||
|
||||
@@ -55,6 +55,7 @@ import org.talend.core.language.ECodeLanguage;
|
||||
import org.talend.core.language.LanguageManager;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.model.general.Project;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.core.ui.component.ComponentPaletteUtilities;
|
||||
import org.talend.core.ui.component.ComponentsFactoryProvider;
|
||||
import org.talend.designer.components.exchange.ExchangePlugin;
|
||||
@@ -205,14 +206,19 @@ public class ExchangeUtils {
|
||||
* @return
|
||||
*/
|
||||
public static File getComponentFolder(String componentfolder) {
|
||||
URL url = FileLocator.find(ExchangePlugin.getDefault().getBundle(), new Path(componentfolder), null);
|
||||
try {
|
||||
URL fileUrl = FileLocator.toFileURL(url);
|
||||
return new File(fileUrl.getPath());
|
||||
} catch (Exception e) {
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
return null;
|
||||
if (SharedStudioUtils.isSharedStudioMode()) {
|
||||
File componentFolder = SharedStudioUtils.getSharedStudioComponentsExtFolder();
|
||||
return new File (componentFolder, componentfolder);
|
||||
} else {
|
||||
URL url = FileLocator.find(ExchangePlugin.getDefault().getBundle(), new Path(componentfolder), null);
|
||||
try {
|
||||
URL fileUrl = FileLocator.toFileURL(url);
|
||||
return new File(fileUrl.getPath());
|
||||
} catch (Exception e) {
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>filecopy</artifactId>
|
||||
<version>2.0.0</version>
|
||||
<version>2.0.1</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend-copy</name>
|
||||
|
||||
@@ -15,7 +15,10 @@ package org.talend;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.nio.file.attribute.FileTime;
|
||||
|
||||
/**
|
||||
* DOC Administrator class global comment. Detailled comment
|
||||
@@ -35,14 +38,17 @@ public class FileCopy {
|
||||
* @throws IOException : if IO pb.
|
||||
*/
|
||||
public static void copyFile(String srcFileName, String desFileName, boolean delSrc) throws IOException {
|
||||
final File source = new File(srcFileName);
|
||||
final File destination = new File(desFileName);
|
||||
final Path source = Paths.get(srcFileName);
|
||||
final Path destination = Paths.get(desFileName);
|
||||
|
||||
if (delSrc) {
|
||||
// move : more efficient if in same FS and mustr delete existing file.
|
||||
Files.move(source.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
FileTime lastModifiedTime = Files.getLastModifiedTime(source);
|
||||
Files.move(source, destination, StandardCopyOption.REPLACE_EXISTING);
|
||||
Files.setLastModifiedTime(destination,lastModifiedTime);
|
||||
} else {
|
||||
Files.copy(source.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING);
|
||||
Files.setLastModifiedTime(destination,Files.getLastModifiedTime(source));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -100,6 +100,24 @@ class FileCopyTest {
|
||||
Assertions.assertEquals(referenceSize, copy.length(), "Size error");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLastModifiedTime() throws Exception {
|
||||
final URL repCopy = Thread.currentThread().getContextClassLoader().getResource("copy");
|
||||
|
||||
File file = this.buildFile("fileLMT.txt", 10L * 1024L);
|
||||
file.deleteOnExit();
|
||||
long referencceTime = 324723894L;
|
||||
file.setLastModified(referencceTime);
|
||||
|
||||
File copy = new File(repCopy.getPath(), "fileLMTDestination.txt");
|
||||
if (copy.exists()) {
|
||||
copy.delete();
|
||||
}
|
||||
copy.deleteOnExit();
|
||||
FileCopy.copyFile(file.getPath(), copy.getPath(), true);
|
||||
Assertions.assertEquals(referencceTime, copy.lastModified(), "modified time is not idential");
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a new file for testing.
|
||||
*
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>simpleexcel-2.2-20190722</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>simpleexcel</artifactId>
|
||||
<version>2.4-20200923</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>simpleexcel</name>
|
||||
@@ -13,7 +13,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<java.source.version>1.6</java.source.version>
|
||||
<java.source.version>1.8</java.source.version>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
@@ -43,47 +43,30 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi</artifactId>
|
||||
<version>4.1.0</version>
|
||||
<version>4.1.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-scratchpad</artifactId>
|
||||
<version>4.1.0</version>
|
||||
<version>4.1.2</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml</artifactId>
|
||||
<version>4.1.0</version>
|
||||
<version>4.1.2</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml-schemas -->
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml-schemas</artifactId>
|
||||
<version>4.1.0</version>
|
||||
<version>4.1.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.geronimo.specs/geronimo-stax-api_1.0_spec -->
|
||||
<dependency>
|
||||
<groupId>org.apache.geronimo.specs</groupId>
|
||||
<artifactId>geronimo-stax-api_1.0_spec</artifactId>
|
||||
<version>1.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dom4j</groupId>
|
||||
<artifactId>dom4j</artifactId>
|
||||
<version>2.1.3</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.xmlbeans/xmlbeans -->
|
||||
<dependency>
|
||||
<groupId>org.apache.xmlbeans</groupId>
|
||||
<artifactId>xmlbeans</artifactId>
|
||||
<version>3.1.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<resources>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// ============================================================================
|
||||
//
|
||||
// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
|
||||
//
|
||||
// This source code is available under agreement available at
|
||||
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// ============================================================================
|
||||
//
|
||||
// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
|
||||
//
|
||||
// This source code is available under agreement available at
|
||||
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// ============================================================================
|
||||
//
|
||||
// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
|
||||
//
|
||||
// This source code is available under agreement available at
|
||||
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// ============================================================================
|
||||
//
|
||||
// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
|
||||
//
|
||||
// This source code is available under agreement available at
|
||||
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// ============================================================================
|
||||
//
|
||||
// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
|
||||
//
|
||||
// This source code is available under agreement available at
|
||||
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
|
||||
@@ -3,14 +3,13 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>components-soap</artifactId>
|
||||
<version>2.2-20200730</version>
|
||||
<version>2.3-20200918</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend-soap</name>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<cxf.version>3.1.1</cxf.version>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
</properties>
|
||||
|
||||
@@ -46,19 +45,14 @@
|
||||
<systemPath>${java.home}/lib/rt.jar</systemPath>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>jdom</groupId>
|
||||
<artifactId>jdom</artifactId>
|
||||
<version>1.1</version>
|
||||
<groupId>org.dom4j</groupId>
|
||||
<artifactId>dom4j</artifactId>
|
||||
<version>2.1.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.sun.xml.messaging.saaj</groupId>
|
||||
<artifactId>saaj-impl</artifactId>
|
||||
<version>1.3.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>javax.activation</groupId>
|
||||
<artifactId>activation</artifactId>
|
||||
<version>1.1</version>
|
||||
<groupId>com.sun.xml.messaging.saaj</groupId>
|
||||
<artifactId>saaj-impl</artifactId>
|
||||
<version>1.5.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>xerces</groupId>
|
||||
@@ -68,7 +62,7 @@
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<version>1.9</version>
|
||||
<version>1.14</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
@@ -108,4 +102,4 @@
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
</project>
|
||||
|
||||
@@ -32,8 +32,7 @@ import javax.xml.transform.stream.StreamResult;
|
||||
import javax.xml.transform.stream.StreamSource;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.jdom.input.DOMBuilder;
|
||||
import org.jdom.output.XMLOutputter;
|
||||
import org.dom4j.io.DOMReader;
|
||||
import org.talend.soap.sun.SunNtlmAuthenticationUpdater;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
@@ -45,8 +44,6 @@ public class SOAPUtil {
|
||||
|
||||
private static final String vmVendor = System.getProperty("java.vendor.url");
|
||||
|
||||
private static final String ibmVmVendor = "http://www.ibm.com/";
|
||||
|
||||
private static final String sunVmVendor = "http://java.sun.com/";
|
||||
|
||||
private static final String oracleVmVendor = "http://java.oracle.com/";
|
||||
@@ -140,12 +137,7 @@ public class SOAPUtil {
|
||||
StreamSource preppedMsgSrc = new StreamSource(stream);
|
||||
soapPart.setContent(preppedMsgSrc);
|
||||
|
||||
// InputStream stream = new FileInputStream(new File("d://soap.txt"));
|
||||
// StreamSource preppedMsgSrc = new StreamSource(stream);
|
||||
// soapPart.setContent(preppedMsgSrc);
|
||||
|
||||
message.saveChanges();
|
||||
// Send the message
|
||||
|
||||
SOAPMessage reply = connection.call(message, destination);
|
||||
|
||||
@@ -226,7 +218,7 @@ public class SOAPUtil {
|
||||
Node content;
|
||||
Element headerRootElem = document.createElement("Header");
|
||||
|
||||
Iterator childElements = header.getChildElements();
|
||||
Iterator<javax.xml.soap.Node> childElements = header.getChildElements();
|
||||
org.w3c.dom.Node domNode = null;
|
||||
while (childElements.hasNext()) {
|
||||
domNode = (org.w3c.dom.Node) childElements.next();
|
||||
@@ -245,12 +237,11 @@ public class SOAPUtil {
|
||||
return reHeaderMessage;
|
||||
}
|
||||
|
||||
private String Doc2StringWithoutDeclare(Document doc) {
|
||||
DOMBuilder builder = new DOMBuilder();
|
||||
org.jdom.Document jdomDoc = builder.build(doc);
|
||||
XMLOutputter outputter = new XMLOutputter();
|
||||
return outputter.outputString(jdomDoc.getRootElement());
|
||||
}
|
||||
private String Doc2StringWithoutDeclare(Document doc) {
|
||||
DOMReader reader = new DOMReader();
|
||||
org.dom4j.Document document = reader.read(doc);
|
||||
return document.getRootElement().asXML();
|
||||
}
|
||||
|
||||
/**
|
||||
* invoke soap and return the response document
|
||||
@@ -363,4 +354,4 @@ public class SOAPUtil {
|
||||
headers.setHeader("Authorization", "Basic " + encodeUserInfo);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>talend-aws</artifactId>
|
||||
<version>1.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend-aws</name>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<java.source.version>1.8</java.source.version>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
</snapshotRepository>
|
||||
<repository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk</artifactId>
|
||||
<version>1.11.848</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/java</directory>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>${java.source.version}</source>
|
||||
<target>${java.source.version}</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@@ -0,0 +1,277 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import static com.amazonaws.event.SDKProgressPublisher.publishProgress;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedList;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import com.amazonaws.AmazonClientException;
|
||||
import com.amazonaws.AmazonServiceException;
|
||||
import com.amazonaws.event.ProgressEventType;
|
||||
import com.amazonaws.event.ProgressListener;
|
||||
import com.amazonaws.event.ProgressListenerChain;
|
||||
import com.amazonaws.services.s3.model.LegacyS3ProgressListener;
|
||||
import com.amazonaws.services.s3.transfer.Transfer;
|
||||
import com.amazonaws.services.s3.transfer.TransferProgress;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferMonitor;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
|
||||
|
||||
|
||||
/**
|
||||
* Abstract transfer implementation.
|
||||
*/
|
||||
public abstract class AbstractTransfer implements Transfer {
|
||||
|
||||
/** The current state of this transfer. */
|
||||
protected volatile TransferState state = TransferState.Waiting;
|
||||
|
||||
protected TransferMonitor monitor;
|
||||
|
||||
/** The progress of this transfer. */
|
||||
private final TransferProgress transferProgress;
|
||||
|
||||
private final String description;
|
||||
|
||||
/** Hook for adding/removing more progress listeners. */
|
||||
protected final ProgressListenerChain listenerChain;
|
||||
|
||||
/** Collection of listeners to be notified for changes to the state of this transfer via setState() */
|
||||
protected final Collection<TransferStateChangeListener> stateChangeListeners = new LinkedList<TransferStateChangeListener>();
|
||||
|
||||
AbstractTransfer(String description, TransferProgress transferProgress, ProgressListenerChain progressListenerChain) {
|
||||
this(description, transferProgress, progressListenerChain, null);
|
||||
}
|
||||
|
||||
AbstractTransfer(String description, TransferProgress transferProgress,
|
||||
ProgressListenerChain progressListenerChain, TransferStateChangeListener stateChangeListener) {
|
||||
this.description = description;
|
||||
this.listenerChain = progressListenerChain;
|
||||
this.transferProgress = transferProgress;
|
||||
addStateChangeListener(stateChangeListener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether or not the transfer is finished (i.e. completed successfully,
|
||||
* failed, or was canceled). This method should never block.
|
||||
*
|
||||
* @return Returns <code>true</code> if this transfer is finished (i.e. completed successfully,
|
||||
* failed, or was canceled). Returns <code>false</code> if otherwise.
|
||||
*/
|
||||
public final synchronized boolean isDone() {
|
||||
return (state == TransferState.Failed ||
|
||||
state == TransferState.Completed ||
|
||||
state == TransferState.Canceled);
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for this transfer to complete. This is a blocking call; the current
|
||||
* thread is suspended until this transfer completes.
|
||||
*
|
||||
* @throws AmazonClientException
|
||||
* If any errors were encountered in the client while making the
|
||||
* request or handling the response.
|
||||
* @throws AmazonServiceException
|
||||
* If any errors occurred in Amazon S3 while processing the
|
||||
* request.
|
||||
* @throws InterruptedException
|
||||
* If this thread is interrupted while waiting for the transfer
|
||||
* to complete.
|
||||
*/
|
||||
public void waitForCompletion()
|
||||
throws AmazonClientException, AmazonServiceException, InterruptedException {
|
||||
try {
|
||||
Object result = null;
|
||||
while (!monitor.isDone() || result == null) {
|
||||
Future<?> f = monitor.getFuture();
|
||||
result = f.get();
|
||||
}
|
||||
} catch (ExecutionException e) {
|
||||
rethrowExecutionException(e);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for this transfer to finish and returns any error that occurred, or
|
||||
* returns <code>null</code> if no errors occurred.
|
||||
* This is a blocking call; the current thread
|
||||
* will be suspended until this transfer either fails or completes
|
||||
* successfully.
|
||||
*
|
||||
* @return Any error that occurred while processing this transfer.
|
||||
* Otherwise returns <code>null</code> if no errors occurred.
|
||||
*
|
||||
* @throws InterruptedException
|
||||
* If this thread is interrupted while waiting for the transfer
|
||||
* to complete.
|
||||
*/
|
||||
public AmazonClientException waitForException() throws InterruptedException {
|
||||
try {
|
||||
|
||||
/**
|
||||
* Do not remove the while loop. We need this as the future returned by
|
||||
* monitor.getFuture() is set two times during the upload and copy operations.
|
||||
*/
|
||||
while (!monitor.isDone()) {
|
||||
monitor.getFuture().get();
|
||||
}
|
||||
monitor.getFuture().get();
|
||||
return null;
|
||||
} catch (ExecutionException e) {
|
||||
return unwrapExecutionException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a human-readable description of this transfer.
|
||||
*
|
||||
* @return A human-readable description of this transfer.
|
||||
*/
|
||||
public String getDescription() {
|
||||
return description;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current state of this transfer.
|
||||
*
|
||||
* @return The current state of this transfer.
|
||||
*/
|
||||
public synchronized TransferState getState() {
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the current state of this transfer.
|
||||
*/
|
||||
public void setState(TransferState state) {
|
||||
synchronized (this) {
|
||||
this.state = state;
|
||||
}
|
||||
for ( TransferStateChangeListener listener : stateChangeListeners ) {
|
||||
listener.transferStateChanged(this, state);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Notifies all the registered state change listeners of the state update.
|
||||
*/
|
||||
public void notifyStateChangeListeners(TransferState state) {
|
||||
for ( TransferStateChangeListener listener : stateChangeListeners ) {
|
||||
listener.transferStateChanged(this, state);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the specified progress listener to the list of listeners
|
||||
* receiving updates about this transfer's progress.
|
||||
*
|
||||
* @param listener
|
||||
* The progress listener to add.
|
||||
*/
|
||||
public synchronized void addProgressListener(ProgressListener listener) {
|
||||
listenerChain.addProgressListener(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the specified progress listener from the list of progress
|
||||
* listeners receiving updates about this transfer's progress.
|
||||
*
|
||||
* @param listener
|
||||
* The progress listener to remove.
|
||||
*/
|
||||
public synchronized void removeProgressListener(ProgressListener listener) {
|
||||
listenerChain.removeProgressListener(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Replaced by {@link #addProgressListener(ProgressListener)}
|
||||
*/
|
||||
@Deprecated
|
||||
public synchronized void addProgressListener(com.amazonaws.services.s3.model.ProgressListener listener) {
|
||||
listenerChain.addProgressListener(new LegacyS3ProgressListener(listener));
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Replaced by {@link #removeProgressListener(ProgressListener)}
|
||||
*/
|
||||
@Deprecated
|
||||
public synchronized void removeProgressListener(com.amazonaws.services.s3.model.ProgressListener listener) {
|
||||
listenerChain.removeProgressListener(new LegacyS3ProgressListener(listener));
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the given state change listener to the collection of listeners.
|
||||
*/
|
||||
public synchronized void addStateChangeListener(TransferStateChangeListener listener) {
|
||||
if ( listener != null )
|
||||
stateChangeListeners.add(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the given state change listener from the collection of listeners.
|
||||
*/
|
||||
public synchronized void removeStateChangeListener(TransferStateChangeListener listener) {
|
||||
if ( listener != null )
|
||||
stateChangeListeners.remove(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns progress information about this transfer.
|
||||
*
|
||||
* @return The progress information about this transfer.
|
||||
*/
|
||||
public TransferProgress getProgress() {
|
||||
return transferProgress;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the monitor used to poll for transfer completion.
|
||||
*/
|
||||
public void setMonitor(TransferMonitor monitor) {
|
||||
this.monitor = monitor;
|
||||
}
|
||||
|
||||
public TransferMonitor getMonitor() {
|
||||
return monitor;
|
||||
}
|
||||
|
||||
protected void fireProgressEvent(final ProgressEventType eventType) {
|
||||
publishProgress(listenerChain, eventType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Examines the cause of the specified ExecutionException and either
|
||||
* rethrows it directly (if it's a type of AmazonClientException) or wraps
|
||||
* it in an AmazonClientException and rethrows it.
|
||||
*
|
||||
* @param e
|
||||
* The execution exception to examine.
|
||||
*/
|
||||
protected void rethrowExecutionException(ExecutionException e) {
|
||||
throw unwrapExecutionException(e);
|
||||
}
|
||||
|
||||
/**
|
||||
* Unwraps the root exception that caused the specified ExecutionException
|
||||
* and returns it. If it was not an instance of AmazonClientException, it is
|
||||
* wrapped as an AmazonClientException.
|
||||
*
|
||||
* @param e
|
||||
* The ExecutionException to unwrap.
|
||||
*
|
||||
* @return The root exception that caused the specified ExecutionException.
|
||||
*/
|
||||
protected AmazonClientException unwrapExecutionException(ExecutionException e) {
|
||||
Throwable t = e;
|
||||
while (t.getCause() != null && t instanceof ExecutionException) {
|
||||
t = t.getCause();
|
||||
}
|
||||
if (t instanceof AmazonClientException) {
|
||||
return (AmazonClientException) t;
|
||||
}
|
||||
return new AmazonClientException("Unable to complete transfer: " + t.getMessage(), t);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.annotation.SdkInternalApi;
|
||||
import com.amazonaws.services.s3.internal.ServiceUtils;
|
||||
import com.amazonaws.services.s3.transfer.Transfer;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
/**
|
||||
* Helper class to merge all the individual part files into a destinationFile.
|
||||
*/
|
||||
@SdkInternalApi
|
||||
public class CompleteMultipartDownload implements Callable<File> {
|
||||
private final List<Future<File>> partFiles;
|
||||
private final File destinationFile;
|
||||
private final DownloadImpl download;
|
||||
private Integer currentPartNumber;
|
||||
|
||||
public CompleteMultipartDownload(List<Future<File>> files, File destinationFile, DownloadImpl download, Integer currentPartNumber) {
|
||||
this.partFiles = files;
|
||||
this.destinationFile = destinationFile;
|
||||
this.download = download;
|
||||
this.currentPartNumber = currentPartNumber;
|
||||
}
|
||||
|
||||
@Override
|
||||
public File call() throws Exception {
|
||||
for (Future<File> file : partFiles) {
|
||||
ServiceUtils.appendFile(file.get(), destinationFile);
|
||||
download.updatePersistableTransfer(currentPartNumber++);
|
||||
}
|
||||
|
||||
download.setState(Transfer.TransferState.Completed);
|
||||
return destinationFile;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import com.amazonaws.services.s3.model.CryptoMode;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.transfer.Transfer;
|
||||
import com.amazonaws.services.s3.transfer.exception.PauseException;
|
||||
|
||||
/**
|
||||
* Represents an asynchronous download from Amazon S3.
|
||||
*/
|
||||
public interface Download extends Transfer {
|
||||
|
||||
/**
|
||||
* Returns the ObjectMetadata for the object being downloaded.
|
||||
*
|
||||
* @return The ObjectMetadata for the object being downloaded.
|
||||
*/
|
||||
public ObjectMetadata getObjectMetadata();
|
||||
|
||||
/**
|
||||
* The name of the bucket where the object is being downloaded from.
|
||||
*
|
||||
* @return The name of the bucket where the object is being downloaded from.
|
||||
*/
|
||||
public String getBucketName();
|
||||
|
||||
/**
|
||||
* The key under which this object was stored in Amazon S3.
|
||||
*
|
||||
* @return The key under which this object was stored in Amazon S3.
|
||||
*/
|
||||
public String getKey();
|
||||
|
||||
/**
|
||||
* Cancels this download.
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
public void abort() throws IOException;
|
||||
|
||||
/**
|
||||
* Pause the current download operation and returns the information that can
|
||||
* be used to resume the download at a later time.
|
||||
*
|
||||
* Resuming a download would not perform ETag check as range get is
|
||||
* performed for downloading the object's remaining contents.
|
||||
*
|
||||
* Resuming a download for an object encrypted using
|
||||
* {@link CryptoMode#StrictAuthenticatedEncryption} would result in
|
||||
* AmazonClientException as authenticity cannot be guaranteed for a range
|
||||
* get operation.
|
||||
*
|
||||
* @throws PauseException
|
||||
* If any errors were encountered while trying to pause the
|
||||
* download.
|
||||
*/
|
||||
public PersistableDownload pause() throws PauseException;
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.net.SocketException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import javax.net.ssl.SSLProtocolException;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import com.amazonaws.AmazonClientException;
|
||||
import com.amazonaws.SdkClientException;
|
||||
import com.amazonaws.annotation.SdkInternalApi;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.internal.FileLocks;
|
||||
import com.amazonaws.services.s3.internal.ServiceUtils;
|
||||
import com.amazonaws.services.s3.internal.ServiceUtils.RetryableS3DownloadTask;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.amazonaws.services.s3.transfer.Transfer.TransferState;
|
||||
import com.amazonaws.services.s3.transfer.exception.FileLockException;
|
||||
import com.amazonaws.util.IOUtils;
|
||||
|
||||
@SdkInternalApi
|
||||
final class DownloadCallable implements Callable<File> {
|
||||
private static final Log LOG = LogFactory.getLog(DownloadCallable.class);
|
||||
|
||||
private final AmazonS3 s3;
|
||||
private final CountDownLatch latch;
|
||||
private final GetObjectRequest req;
|
||||
private final boolean resumeExistingDownload;
|
||||
private final DownloadImpl download;
|
||||
private final File dstfile;
|
||||
private final long origStartingByte;
|
||||
private final long timeout;
|
||||
private final ScheduledExecutorService timedExecutor;
|
||||
/** The thread pool in which parts are downloaded downloaded. */
|
||||
private final ExecutorService executor;
|
||||
private final List<Future<File>> futureFiles;
|
||||
private final boolean isDownloadParallel;
|
||||
private Integer lastFullyMergedPartNumber;
|
||||
private final boolean resumeOnRetry;
|
||||
|
||||
private long expectedFileLength;
|
||||
|
||||
DownloadCallable(AmazonS3 s3, CountDownLatch latch,
|
||||
GetObjectRequest req, boolean resumeExistingDownload,
|
||||
DownloadImpl download, File dstfile, long origStartingByte,
|
||||
long expectedFileLength, long timeout,
|
||||
ScheduledExecutorService timedExecutor,
|
||||
ExecutorService executor,
|
||||
Integer lastFullyDownloadedPartNumber, boolean isDownloadParallel, boolean resumeOnRetry)
|
||||
{
|
||||
if (s3 == null || latch == null || req == null || dstfile == null || download == null)
|
||||
throw new IllegalArgumentException();
|
||||
this.s3 = s3;
|
||||
this.latch = latch;
|
||||
this.req = req;
|
||||
this.resumeExistingDownload = resumeExistingDownload;
|
||||
this.download = download;
|
||||
this.dstfile = dstfile;
|
||||
this.origStartingByte = origStartingByte;
|
||||
this.expectedFileLength = expectedFileLength;
|
||||
this.timeout = timeout;
|
||||
this.timedExecutor = timedExecutor;
|
||||
this.executor = executor;
|
||||
this.futureFiles = new ArrayList<Future<File>>();
|
||||
this.lastFullyMergedPartNumber = lastFullyDownloadedPartNumber;
|
||||
this.isDownloadParallel = isDownloadParallel;
|
||||
this.resumeOnRetry = resumeOnRetry;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method must return a non-null object, or else the existing
|
||||
* implementation in {@link AbstractTransfer#waitForCompletion()}
|
||||
* would block forever.
|
||||
*
|
||||
* @return the downloaded file
|
||||
*/
|
||||
@Override
|
||||
public File call() throws Exception {
|
||||
try {
|
||||
latch.await();
|
||||
|
||||
if (isTimeoutEnabled()) {
|
||||
timedExecutor.schedule(new Runnable() {
|
||||
public void run() {
|
||||
try {
|
||||
if (download.getState() != TransferState.Completed) {
|
||||
download.abort();
|
||||
}
|
||||
} catch(Exception e) {
|
||||
throw new SdkClientException(
|
||||
"Unable to abort download after timeout", e);
|
||||
}
|
||||
}
|
||||
}, timeout, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
download.setState(TransferState.InProgress);
|
||||
ServiceUtils.createParentDirectoryIfNecessary(dstfile);
|
||||
|
||||
if (isDownloadParallel) {
|
||||
downloadInParallel(ServiceUtils.getPartCount(req, s3));
|
||||
} else {
|
||||
S3Object s3Object = retryableDownloadS3ObjectToFile(dstfile,
|
||||
new DownloadTaskImpl(s3, download, req));
|
||||
updateDownloadStatus(s3Object);
|
||||
}
|
||||
return dstfile;
|
||||
} catch (Throwable t) {
|
||||
// Cancel all the futures
|
||||
for (Future<File> f : futureFiles) {
|
||||
f.cancel(true);
|
||||
}
|
||||
// Downloads aren't allowed to move from canceled to failed
|
||||
if (download.getState() != TransferState.Canceled) {
|
||||
download.setState(TransferState.Failed);
|
||||
}
|
||||
if (t instanceof Exception)
|
||||
throw (Exception) t;
|
||||
else
|
||||
throw (Error) t;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes the result from serial download,
|
||||
* updates the transfer state and monitor in downloadImpl object
|
||||
* based on the result.
|
||||
*/
|
||||
private void updateDownloadStatus(S3Object result) {
|
||||
if (result == null) {
|
||||
download.setState(TransferState.Canceled);
|
||||
download.setMonitor(new DownloadMonitor(download, null));
|
||||
} else {
|
||||
download.setState(TransferState.Completed);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads each part of the object into a separate file synchronously and
|
||||
* combines all the files into a single file.
|
||||
*/
|
||||
private void downloadInParallel(int partCount) throws Exception {
|
||||
if (lastFullyMergedPartNumber == null) {
|
||||
lastFullyMergedPartNumber = 0;
|
||||
}
|
||||
|
||||
for (int i = lastFullyMergedPartNumber + 1; i <= partCount; i++) {
|
||||
GetObjectRequest getPartRequest = new GetObjectRequest(req.getBucketName(), req.getKey(),
|
||||
req.getVersionId()).withUnmodifiedSinceConstraint(req.getUnmodifiedSinceConstraint())
|
||||
.withModifiedSinceConstraint(req.getModifiedSinceConstraint())
|
||||
.withResponseHeaders(req.getResponseHeaders()).withSSECustomerKey(req.getSSECustomerKey())
|
||||
.withGeneralProgressListener(req.getGeneralProgressListener());
|
||||
|
||||
getPartRequest.setMatchingETagConstraints(req.getMatchingETagConstraints());
|
||||
getPartRequest.setNonmatchingETagConstraints(req.getNonmatchingETagConstraints());
|
||||
getPartRequest.setRequesterPays(req.isRequesterPays());
|
||||
|
||||
futureFiles.add(
|
||||
executor.submit(new DownloadPartCallable(s3, getPartRequest.withPartNumber(i), dstfile)));
|
||||
}
|
||||
|
||||
truncateDestinationFileIfNecessary();
|
||||
Future<File> future = executor.submit(new CompleteMultipartDownload(futureFiles, dstfile, download, ++lastFullyMergedPartNumber));
|
||||
((DownloadMonitor) download.getMonitor()).setFuture(future);
|
||||
}
|
||||
|
||||
/**
|
||||
* If only partial part object is merged into the dstFile(due to pause
|
||||
* operation), adjust the file length so that the part starts writing from
|
||||
* the correct position.
|
||||
*/
|
||||
private void truncateDestinationFileIfNecessary() {
|
||||
RandomAccessFile raf = null;
|
||||
if (!FileLocks.lock(dstfile)) {
|
||||
throw new FileLockException("Fail to lock " + dstfile);
|
||||
}
|
||||
|
||||
try {
|
||||
raf = new RandomAccessFile(dstfile, "rw");
|
||||
if (lastFullyMergedPartNumber == 0) {
|
||||
raf.setLength(0);
|
||||
} else {
|
||||
long lastByte = ServiceUtils.getLastByteInPart(s3, req, lastFullyMergedPartNumber);
|
||||
if (dstfile.length() < lastByte) {
|
||||
throw new SdkClientException(
|
||||
"File " + dstfile.getAbsolutePath() + " has been modified since last pause.");
|
||||
}
|
||||
raf.setLength(lastByte + 1);
|
||||
download.getProgress().updateProgress(lastByte + 1);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new SdkClientException("Unable to append part file to dstfile " + e.getMessage(), e);
|
||||
} finally {
|
||||
IOUtils.closeQuietly(raf, LOG);
|
||||
FileLocks.unlock(dstfile);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is called only if it is a resumed download.
|
||||
*
|
||||
* Adjust the range of the get request, and the expected (ie current) file
|
||||
* length of the destination file to append to.
|
||||
*/
|
||||
private void adjustRequest(GetObjectRequest req) {
|
||||
long[] range = req.getRange();
|
||||
long lastByte = range[1];
|
||||
long totalBytesToDownload = lastByte - this.origStartingByte + 1;
|
||||
|
||||
if (dstfile.exists()) {
|
||||
if (!FileLocks.lock(dstfile)) {
|
||||
throw new FileLockException("Fail to lock " + dstfile
|
||||
+ " for range adjustment");
|
||||
}
|
||||
try {
|
||||
expectedFileLength = dstfile.length();
|
||||
long startingByte = this.origStartingByte + expectedFileLength;
|
||||
LOG.info("Adjusting request range from " + Arrays.toString(range)
|
||||
+ " to "
|
||||
+ Arrays.toString(new long[] { startingByte, lastByte })
|
||||
+ " for file " + dstfile);
|
||||
req.setRange(startingByte, lastByte);
|
||||
totalBytesToDownload = lastByte - startingByte + 1;
|
||||
} finally {
|
||||
FileLocks.unlock(dstfile);
|
||||
}
|
||||
}
|
||||
|
||||
if (totalBytesToDownload < 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"Unable to determine the range for download operation. lastByte="
|
||||
+ lastByte + ", origStartingByte=" + origStartingByte
|
||||
+ ", expectedFileLength=" + expectedFileLength
|
||||
+ ", totalBytesToDownload=" + totalBytesToDownload);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private S3Object retryableDownloadS3ObjectToFile(File file,
|
||||
RetryableS3DownloadTask retryableS3DownloadTask) {
|
||||
boolean hasRetried = false;
|
||||
S3Object s3Object;
|
||||
for (;;) {
|
||||
final boolean appendData = resumeExistingDownload || (resumeOnRetry && hasRetried);
|
||||
if (appendData && hasRetried) {
|
||||
// Need to adjust the get range or else we risk corrupting the downloaded file
|
||||
adjustRequest(req);
|
||||
}
|
||||
s3Object = retryableS3DownloadTask.getS3ObjectStream();
|
||||
if (s3Object == null)
|
||||
return null;
|
||||
try {
|
||||
if (testing && resumeExistingDownload && !hasRetried) {
|
||||
throw new SdkClientException("testing");
|
||||
}
|
||||
ServiceUtils.downloadToFile(s3Object, file,
|
||||
retryableS3DownloadTask.needIntegrityCheck(),
|
||||
appendData, expectedFileLength);
|
||||
return s3Object;
|
||||
} catch (AmazonClientException ace) {
|
||||
if (!ace.isRetryable())
|
||||
throw ace;
|
||||
// Determine whether an immediate retry is needed according to the captured SdkClientException.
|
||||
// (There are three cases when downloadObjectToFile() throws SdkClientException:
|
||||
// 1) SocketException or SSLProtocolException when writing to disk (e.g. when user aborts the download)
|
||||
// 2) Other IOException when writing to disk
|
||||
// 3) MD5 hashes don't match
|
||||
// For 1) If SocketException is the result of the client side resetting the connection, this is retried
|
||||
// Cases 2) and 3) will always be retried
|
||||
final Throwable cause = ace.getCause();
|
||||
if ((cause instanceof SocketException && !cause.getMessage().equals("Connection reset"))
|
||||
|| (cause instanceof SSLProtocolException)) {
|
||||
throw ace;
|
||||
} else {
|
||||
if (hasRetried)
|
||||
throw ace;
|
||||
else {
|
||||
LOG.info("Retry the download of object " + s3Object.getKey() + " (bucket " + s3Object.getBucketName() + ")", ace);
|
||||
hasRetried = true;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
s3Object.getObjectContent().abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isTimeoutEnabled() {
|
||||
return timeout > 0;
|
||||
}
|
||||
|
||||
private static boolean testing;
|
||||
/**
|
||||
* Used for testing purpose only.
|
||||
*/
|
||||
static void setTesting(boolean b) {
|
||||
testing = b;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import com.amazonaws.annotation.SdkInternalApi;
|
||||
import com.amazonaws.event.ProgressEventType;
|
||||
import com.amazonaws.event.ProgressListenerChain;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.amazonaws.services.s3.transfer.TransferProgress;
|
||||
import com.amazonaws.services.s3.transfer.exception.PauseException;
|
||||
import com.amazonaws.services.s3.transfer.internal.S3ProgressPublisher;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferManagerUtils;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
|
||||
|
||||
public class DownloadImpl extends AbstractTransfer implements Download {
|
||||
private S3Object s3Object;
|
||||
|
||||
/**
|
||||
* Information to resume if the download is paused.
|
||||
*/
|
||||
private PersistableDownload persistableDownload;
|
||||
|
||||
/**
|
||||
* The last part that has been successfully written into the downloaded file.
|
||||
*/
|
||||
private Integer lastFullyDownloadedPartNumber;
|
||||
|
||||
private final GetObjectRequest getObjectRequest;
|
||||
private final File file;
|
||||
private final ObjectMetadata objectMetadata;
|
||||
private final ProgressListenerChain progressListenerChain;
|
||||
|
||||
@Deprecated
|
||||
public DownloadImpl(String description, TransferProgress transferProgress,
|
||||
ProgressListenerChain progressListenerChain, S3Object s3Object, TransferStateChangeListener listener,
|
||||
GetObjectRequest getObjectRequest, File file) {
|
||||
this(description, transferProgress, progressListenerChain, s3Object, listener,
|
||||
getObjectRequest, file, null, false);
|
||||
}
|
||||
|
||||
public DownloadImpl(String description, TransferProgress transferProgress,
|
||||
ProgressListenerChain progressListenerChain, S3Object s3Object, TransferStateChangeListener listener,
|
||||
GetObjectRequest getObjectRequest, File file,
|
||||
ObjectMetadata objectMetadata, boolean isDownloadParallel) {
|
||||
super(description, transferProgress, progressListenerChain, listener);
|
||||
this.s3Object = s3Object;
|
||||
this.objectMetadata = objectMetadata;
|
||||
this.getObjectRequest = getObjectRequest;
|
||||
this.file = file;
|
||||
this.progressListenerChain = progressListenerChain;
|
||||
this.persistableDownload = captureDownloadState(getObjectRequest, file);
|
||||
S3ProgressPublisher.publishTransferPersistable(progressListenerChain, persistableDownload);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the ObjectMetadata for the object being downloaded.
|
||||
*
|
||||
* @return The ObjectMetadata for the object being downloaded.
|
||||
*/
|
||||
public synchronized ObjectMetadata getObjectMetadata() {
|
||||
if (s3Object != null) {
|
||||
return s3Object.getObjectMetadata();
|
||||
}
|
||||
return objectMetadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* The name of the bucket where the object is being downloaded from.
|
||||
*
|
||||
* @return The name of the bucket where the object is being downloaded from.
|
||||
*/
|
||||
public String getBucketName() {
|
||||
return getObjectRequest.getBucketName();
|
||||
}
|
||||
|
||||
/**
|
||||
* The key under which this object was stored in Amazon S3.
|
||||
*
|
||||
* @return The key under which this object was stored in Amazon S3.
|
||||
*/
|
||||
public String getKey() {
|
||||
return getObjectRequest.getKey();
|
||||
}
|
||||
|
||||
/**
|
||||
* Only for internal use.
|
||||
* For parallel downloads, Updates the persistableTransfer each time a
|
||||
* part is successfully merged into download file.
|
||||
* Then notify the listeners that new persistableTransfer is available.
|
||||
*/
|
||||
@SdkInternalApi
|
||||
public void updatePersistableTransfer(Integer lastFullyDownloadedPartNumber) {
|
||||
synchronized (this) {
|
||||
this.lastFullyDownloadedPartNumber = lastFullyDownloadedPartNumber;
|
||||
}
|
||||
|
||||
persistableDownload = captureDownloadState(getObjectRequest, file);
|
||||
S3ProgressPublisher.publishTransferPersistable(progressListenerChain, persistableDownload);
|
||||
}
|
||||
|
||||
/**
|
||||
* For parallel downloads, returns the last part number that was
|
||||
* successfully written into the download file.
|
||||
* Returns null for serial downloads.
|
||||
*/
|
||||
public synchronized Integer getLastFullyDownloadedPartNumber() {
|
||||
return lastFullyDownloadedPartNumber;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancels this download.
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized void abort() throws IOException {
|
||||
|
||||
this.monitor.getFuture().cancel(true);
|
||||
|
||||
if ( s3Object != null ) {
|
||||
s3Object.getObjectContent().abort();
|
||||
}
|
||||
setState(TransferState.Canceled);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancels this download, but skip notifying the state change listeners.
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized void abortWithoutNotifyingStateChangeListener() throws IOException {
|
||||
this.monitor.getFuture().cancel(true);
|
||||
this.state = TransferState.Canceled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the S3 object to download.
|
||||
*/
|
||||
public synchronized void setS3Object(S3Object s3Object) {
|
||||
this.s3Object = s3Object;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is also responsible for firing COMPLETED signal to the
|
||||
* listeners.
|
||||
*/
|
||||
@Override
|
||||
public void setState(TransferState state) {
|
||||
super.setState(state);
|
||||
|
||||
switch (state) {
|
||||
case Completed :
|
||||
fireProgressEvent(ProgressEventType.TRANSFER_COMPLETED_EVENT);
|
||||
break;
|
||||
case Canceled:
|
||||
fireProgressEvent(ProgressEventType.TRANSFER_CANCELED_EVENT);
|
||||
break;
|
||||
case Failed:
|
||||
fireProgressEvent(ProgressEventType.TRANSFER_FAILED_EVENT);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the captured state of the download; or null if it should not be
|
||||
* captured (for security reason).
|
||||
*/
|
||||
private PersistableDownload captureDownloadState(
|
||||
final GetObjectRequest getObjectRequest, final File file) {
|
||||
if (getObjectRequest.getSSECustomerKey() == null) {
|
||||
return new PersistableDownload(
|
||||
getObjectRequest.getBucketName(), getObjectRequest.getKey(),
|
||||
getObjectRequest.getVersionId(), getObjectRequest.getRange(),
|
||||
getObjectRequest.getResponseHeaders(), getObjectRequest.isRequesterPays(),
|
||||
file.getAbsolutePath(), getLastFullyDownloadedPartNumber(),
|
||||
getObjectMetadata().getLastModified().getTime());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
*
|
||||
* @see com.amazonaws.services.s3.transfer.Download#pause()
|
||||
*/
|
||||
@Override
|
||||
public PersistableDownload pause() throws PauseException {
|
||||
boolean forceCancel = true;
|
||||
TransferState currentState = getState();
|
||||
this.monitor.getFuture().cancel(true);
|
||||
|
||||
if (persistableDownload == null) {
|
||||
throw new PauseException(TransferManagerUtils.determinePauseStatus(
|
||||
currentState, forceCancel));
|
||||
}
|
||||
return persistableDownload;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferMonitor;
|
||||
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
public class DownloadMonitor implements TransferMonitor {
|
||||
|
||||
private Future<?> future;
|
||||
private final DownloadImpl download;
|
||||
|
||||
public DownloadMonitor(DownloadImpl download, Future<?> future) {
|
||||
this.download = download;
|
||||
this.future = future;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized Future<?> getFuture() {
|
||||
return future;
|
||||
}
|
||||
|
||||
public synchronized void setFuture(Future<?> future) {
|
||||
this.future = future;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isDone() {
|
||||
return download.isDone();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.util.StringUtils;
|
||||
import java.io.File;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
import com.amazonaws.SdkClientException;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
/**
|
||||
* Helper class to get a part from s3,
|
||||
* write the part data to a temporary file and
|
||||
* return the temporary file.
|
||||
*/
|
||||
public class DownloadPartCallable implements Callable<File> {
|
||||
private static final Log LOG = LogFactory.getLog(DownloadPartCallable.class);
|
||||
private static final String TEMP_FILE_MIDDLE_NAME = ".part.";
|
||||
|
||||
private final AmazonS3 s3;
|
||||
private final GetObjectRequest getPartRequest;
|
||||
private final File destinationFile;
|
||||
private final String destinationFilePath;
|
||||
|
||||
public DownloadPartCallable(AmazonS3 s3, GetObjectRequest getPartRequest, File destinationFile) {
|
||||
this.s3 = s3;
|
||||
this.getPartRequest = getPartRequest;
|
||||
this.destinationFile = destinationFile;
|
||||
this.destinationFilePath = destinationFile.getAbsolutePath();
|
||||
}
|
||||
|
||||
public File call() throws Exception {
|
||||
final File partFile = File.createTempFile(
|
||||
UUID.nameUUIDFromBytes(destinationFile.getName().getBytes(StringUtils.UTF8)).toString(),
|
||||
TEMP_FILE_MIDDLE_NAME + getPartRequest.getPartNumber().toString(),
|
||||
new File(destinationFilePath.substring(0, destinationFilePath.lastIndexOf(File.separator))));
|
||||
try {
|
||||
partFile.deleteOnExit();
|
||||
} catch (SecurityException exception) {
|
||||
LOG.warn("SecurityException denied delete access to file " + partFile.getAbsolutePath());
|
||||
}
|
||||
|
||||
if (s3.getObject(getPartRequest, partFile) == null) {
|
||||
throw new SdkClientException(
|
||||
"There is no object in S3 satisfying this request. The getObject method returned null");
|
||||
}
|
||||
return partFile;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3Encryption;
|
||||
import com.amazonaws.services.s3.internal.ServiceUtils;
|
||||
import com.amazonaws.services.s3.internal.SkipMd5CheckStrategy;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
|
||||
final class DownloadTaskImpl implements
|
||||
ServiceUtils.RetryableS3DownloadTask
|
||||
{
|
||||
private final AmazonS3 s3;
|
||||
private final DownloadImpl download;
|
||||
private final GetObjectRequest getObjectRequest;
|
||||
private final SkipMd5CheckStrategy skipMd5CheckStrategy = SkipMd5CheckStrategy.INSTANCE;
|
||||
|
||||
DownloadTaskImpl(AmazonS3 s3, DownloadImpl download,
|
||||
GetObjectRequest getObjectRequest) {
|
||||
this.s3 = s3;
|
||||
this.download = download;
|
||||
this.getObjectRequest = getObjectRequest;
|
||||
}
|
||||
|
||||
@Override
|
||||
public S3Object getS3ObjectStream() {
|
||||
S3Object s3Object = s3.getObject(getObjectRequest);
|
||||
download.setS3Object(s3Object);
|
||||
return s3Object;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean needIntegrityCheck() {
|
||||
// Don't perform the integrity check if the checksum won't matchup.
|
||||
return !(s3 instanceof AmazonS3Encryption) && !skipMd5CheckStrategy.skipClientSideValidationPerRequest(getObjectRequest);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.services.s3.model.ResponseHeaderOverrides;
|
||||
import com.amazonaws.services.s3.transfer.PersistableTransfer;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
/**
|
||||
* An opaque token that holds some private state and can be used to resume a
|
||||
* paused download operation.
|
||||
*/
|
||||
public final class PersistableDownload extends PersistableTransfer {
|
||||
|
||||
static final String TYPE = "download";
|
||||
|
||||
@JsonProperty
|
||||
private final String pauseType = TYPE;
|
||||
|
||||
/** The bucket name in Amazon S3 from where the object has to be downloaded. */
|
||||
@JsonProperty
|
||||
private final String bucketName;
|
||||
|
||||
/** The name of the object in Amazon S3 that has to be downloaded. */
|
||||
@JsonProperty
|
||||
private final String key;
|
||||
|
||||
/** The version id of the object in Amazon S3 to download. */
|
||||
@JsonProperty
|
||||
private final String versionId;
|
||||
|
||||
/** Optional member indicating the byte range of data to retrieve */
|
||||
@JsonProperty
|
||||
private final long[] range;
|
||||
|
||||
/**
|
||||
* Optional field that overrides headers on the response.
|
||||
*/
|
||||
@JsonProperty
|
||||
private final ResponseHeaderOverrides responseHeaders;
|
||||
|
||||
/**
|
||||
* If enabled, the requester is charged for downloading the data from
|
||||
* Requester Pays Buckets.
|
||||
*/
|
||||
@JsonProperty
|
||||
private final boolean isRequesterPays;
|
||||
|
||||
/**
|
||||
* File where the downloaded data is written.
|
||||
*/
|
||||
@JsonProperty
|
||||
private final String file;
|
||||
|
||||
/**
|
||||
* The last part that has been successfully written into the downloaded file.
|
||||
*/
|
||||
@JsonProperty
|
||||
private final Integer lastFullyDownloadedPartNumber;
|
||||
|
||||
/**
|
||||
* Last Modified/created time on Amazon S3 for this object.
|
||||
*/
|
||||
@JsonProperty
|
||||
private final long lastModifiedTime;
|
||||
|
||||
|
||||
public PersistableDownload() {
|
||||
this(null, null, null, null, null, false, null, null, 0L);
|
||||
}
|
||||
|
||||
public PersistableDownload(
|
||||
@JsonProperty(value = "bucketName") String bucketName,
|
||||
@JsonProperty(value = "key") String key,
|
||||
@JsonProperty(value = "versionId") String versionId,
|
||||
@JsonProperty(value = "range") long[] range,
|
||||
@JsonProperty(value = "responseHeaders") ResponseHeaderOverrides responseHeaders,
|
||||
@JsonProperty(value = "isRequesterPays") boolean isRequesterPays,
|
||||
@JsonProperty(value = "file") String file,
|
||||
@JsonProperty(value = "lastFullyDownloadedPartNumber") Integer lastFullyDownloadedPartNumber,
|
||||
@JsonProperty(value = "lastModifiedTime") long lastModifiedTime) {
|
||||
this.bucketName = bucketName;
|
||||
this.key = key;
|
||||
this.versionId = versionId;
|
||||
this.range = range == null ? null : range.clone();
|
||||
this.responseHeaders = responseHeaders;
|
||||
this.isRequesterPays = isRequesterPays;
|
||||
this.file = file;
|
||||
this.lastFullyDownloadedPartNumber = lastFullyDownloadedPartNumber;
|
||||
this.lastModifiedTime = lastModifiedTime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the name of the bucket.
|
||||
*/
|
||||
String getBucketName() {
|
||||
return bucketName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the name of the object.
|
||||
*/
|
||||
String getKey() {
|
||||
return key;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the version id of the object.
|
||||
*/
|
||||
String getVersionId() {
|
||||
return versionId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the byte range of the object to download.
|
||||
*/
|
||||
long[] getRange() {
|
||||
return range == null ? null : range.clone();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the optional response headers.
|
||||
*/
|
||||
ResponseHeaderOverrides getResponseHeaders() {
|
||||
return responseHeaders;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if RequesterPays is enabled on the Amazon S3 bucket else
|
||||
* false.
|
||||
*/
|
||||
boolean isRequesterPays() {
|
||||
return isRequesterPays;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the file where the object is to be downloaded.
|
||||
*/
|
||||
String getFile() {
|
||||
return file;
|
||||
}
|
||||
|
||||
String getPauseType() {
|
||||
return pauseType;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the last part number that was successfully written into the downloaded file.
|
||||
*/
|
||||
Integer getLastFullyDownloadedPartNumber() {
|
||||
return lastFullyDownloadedPartNumber;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the last modified/created time of the object represented by
|
||||
* the bucketName and key.
|
||||
*/
|
||||
Long getlastModifiedTime() {
|
||||
return lastModifiedTime;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.event.ProgressEvent;
|
||||
import com.amazonaws.event.ProgressEventFilter;
|
||||
import com.amazonaws.event.ProgressEventType;
|
||||
|
||||
final class TransferCompletionFilter implements ProgressEventFilter {
|
||||
@Override
|
||||
public ProgressEvent filter(ProgressEvent progressEvent) {
|
||||
// Block COMPLETE events from the low-level GetObject operation,
|
||||
// but we still want to keep the BytesTransferred
|
||||
return progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT
|
||||
? null // discard this event
|
||||
: progressEvent
|
||||
;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.AmazonClientException;
|
||||
import com.amazonaws.AmazonWebServiceRequest;
|
||||
import com.amazonaws.event.ProgressListenerChain;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.internal.FileLocks;
|
||||
import com.amazonaws.services.s3.internal.RequestCopyUtils;
|
||||
import com.amazonaws.services.s3.internal.ServiceUtils;
|
||||
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.transfer.TransferManagerConfiguration;
|
||||
import com.amazonaws.services.s3.transfer.TransferProgress;
|
||||
import com.amazonaws.services.s3.transfer.exception.FileLockException;
|
||||
import com.amazonaws.services.s3.transfer.internal.S3ProgressListener;
|
||||
import com.amazonaws.services.s3.transfer.internal.S3ProgressListenerChain;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferManagerUtils;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferProgressUpdatingListener;
|
||||
import com.amazonaws.util.VersionInfoUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Date;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
public class TransferManager {
|
||||
|
||||
private static final Log log = LogFactory.getLog(TransferManager.class);
|
||||
|
||||
private final AmazonS3 s3;
|
||||
private final ExecutorService executorService;
|
||||
private final TransferManagerConfiguration configuration;
|
||||
private final boolean shutDownThreadPools;
|
||||
|
||||
public TransferManager(AmazonS3 s3) {
|
||||
this.s3 = s3;
|
||||
this.executorService = TransferManagerUtils.createDefaultExecutorService();
|
||||
this.configuration = resolveConfiguration();
|
||||
this.shutDownThreadPools = true;
|
||||
}
|
||||
|
||||
private TransferManagerConfiguration resolveConfiguration() {
|
||||
TransferManagerConfiguration configuration = new TransferManagerConfiguration();
|
||||
configuration.setDisableParallelDownloads(false);
|
||||
return configuration;
|
||||
}
|
||||
|
||||
public Download download(GetObjectRequest getObjectRequest, File file, S3ProgressListener progressListener,
|
||||
long timeoutMillis, boolean resumeOnRetry) {
|
||||
return doDownload(getObjectRequest, file, null, progressListener, ServiceUtils.OVERWRITE_MODE, timeoutMillis, null, 0L,
|
||||
resumeOnRetry);
|
||||
}
|
||||
|
||||
private Download doDownload(final GetObjectRequest getObjectRequest,
|
||||
final File file, final TransferStateChangeListener stateListener,
|
||||
final S3ProgressListener s3progressListener,
|
||||
final boolean resumeExistingDownload,
|
||||
final long timeoutMillis,
|
||||
final Integer lastFullyDownloadedPart,
|
||||
final long lastModifiedTimeRecordedDuringPause,
|
||||
final boolean resumeOnRetry)
|
||||
{
|
||||
assertParameterNotNull(getObjectRequest,
|
||||
"A valid GetObjectRequest must be provided to initiate download");
|
||||
assertParameterNotNull(file,
|
||||
"A valid file must be provided to download into");
|
||||
|
||||
appendSingleObjectUserAgent(getObjectRequest);
|
||||
String description = "Downloading from " + getObjectRequest.getBucketName() + "/" + getObjectRequest.getKey();
|
||||
|
||||
TransferProgress transferProgress = new TransferProgress();
|
||||
// S3 progress listener to capture the persistable transfer when available
|
||||
S3ProgressListenerChain listenerChain = new S3ProgressListenerChain(
|
||||
// The listener for updating transfer progress
|
||||
new TransferProgressUpdatingListener(transferProgress),
|
||||
getObjectRequest.getGeneralProgressListener(),
|
||||
s3progressListener); // Listeners included in the original request
|
||||
// The listener chain used by the low-level GetObject request.
|
||||
// This listener chain ignores any COMPLETE event, so that we could
|
||||
// delay firing the signal until the high-level download fully finishes.
|
||||
getObjectRequest
|
||||
.setGeneralProgressListener(new ProgressListenerChain(new TransferCompletionFilter(), listenerChain));
|
||||
|
||||
GetObjectMetadataRequest getObjectMetadataRequest = RequestCopyUtils.createGetObjectMetadataRequestFrom(getObjectRequest);
|
||||
final ObjectMetadata objectMetadata = s3.getObjectMetadata(getObjectMetadataRequest);
|
||||
|
||||
// Used to check if the object is modified between pause and resume
|
||||
long lastModifiedTime = objectMetadata.getLastModified().getTime();
|
||||
|
||||
long startingByte = 0;
|
||||
long lastByte;
|
||||
|
||||
long[] range = getObjectRequest.getRange();
|
||||
if (range != null && range.length == 2) {
|
||||
startingByte = range[0];
|
||||
lastByte = range[1];
|
||||
} else {
|
||||
lastByte = objectMetadata.getContentLength() - 1;
|
||||
}
|
||||
|
||||
final long origStartingByte = startingByte;
|
||||
final boolean isDownloadParallel = !configuration.isDisableParallelDownloads()
|
||||
&& TransferManagerUtils.isDownloadParallelizable(s3, getObjectRequest, ServiceUtils.getPartCount(getObjectRequest, s3));
|
||||
|
||||
// We still pass the unfiltered listener chain into DownloadImpl
|
||||
final DownloadImpl download = new DownloadImpl(description, transferProgress, listenerChain, null,
|
||||
stateListener, getObjectRequest, file, objectMetadata, isDownloadParallel);
|
||||
|
||||
long totalBytesToDownload = lastByte - startingByte + 1;
|
||||
transferProgress.setTotalBytesToTransfer(totalBytesToDownload);
|
||||
|
||||
// Range information is needed for auto retry of downloads so a retry
|
||||
// request can start at the last downloaded location in the range.
|
||||
//
|
||||
// For obvious reasons, setting a Range header only makes sense if the
|
||||
// object actually has content because it's inclusive, otherwise S3
|
||||
// responds with 4xx
|
||||
//
|
||||
// In addition, we only set the range if the download was *NOT*
|
||||
// determined to be parallelizable above. One of the conditions for
|
||||
// parallel downloads is that getRange() returns null so preserve that.
|
||||
if (totalBytesToDownload > 0 && !isDownloadParallel) {
|
||||
getObjectRequest.withRange(startingByte, lastByte);
|
||||
}
|
||||
|
||||
long fileLength = -1;
|
||||
|
||||
if (resumeExistingDownload) {
|
||||
if (isS3ObjectModifiedSincePause(lastModifiedTime, lastModifiedTimeRecordedDuringPause)) {
|
||||
throw new AmazonClientException("The requested object in bucket " + getObjectRequest.getBucketName()
|
||||
+ " with key " + getObjectRequest.getKey() + " is modified on Amazon S3 since the last pause.");
|
||||
}
|
||||
// There's still a chance the object is modified while the request
|
||||
// is in flight. Set this header so S3 fails the request if this happens.
|
||||
getObjectRequest.setUnmodifiedSinceConstraint(new Date(lastModifiedTime));
|
||||
|
||||
if (!isDownloadParallel) {
|
||||
if (!FileLocks.lock(file)) {
|
||||
throw new FileLockException("Fail to lock " + file + " for resume download");
|
||||
}
|
||||
try {
|
||||
if (file.exists()) {
|
||||
fileLength = file.length();
|
||||
startingByte = startingByte + fileLength;
|
||||
getObjectRequest.setRange(startingByte, lastByte);
|
||||
transferProgress.updateProgress(Math.min(fileLength, totalBytesToDownload));
|
||||
totalBytesToDownload = lastByte - startingByte + 1;
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("Resume download: totalBytesToDownload=" + totalBytesToDownload
|
||||
+ ", origStartingByte=" + origStartingByte + ", startingByte=" + startingByte
|
||||
+ ", lastByte=" + lastByte + ", numberOfBytesRead=" + fileLength + ", file: "
|
||||
+ file);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
FileLocks.unlock(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (totalBytesToDownload < 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"Unable to determine the range for download operation.");
|
||||
}
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(1);
|
||||
Future<?> future = executorService.submit(
|
||||
new DownloadCallable(s3, latch,
|
||||
getObjectRequest, resumeExistingDownload,
|
||||
download, file, origStartingByte, fileLength, timeoutMillis, timedThreadPool,
|
||||
executorService, lastFullyDownloadedPart, isDownloadParallel, resumeOnRetry));
|
||||
download.setMonitor(new DownloadMonitor(download, future));
|
||||
latch.countDown();
|
||||
return download;
|
||||
}
|
||||
|
||||
public void shutdownNow(boolean shutDownS3Client) {
|
||||
if (shutDownThreadPools) {
|
||||
executorService.shutdownNow();
|
||||
timedThreadPool.shutdownNow();
|
||||
}
|
||||
|
||||
if (shutDownS3Client) {
|
||||
s3.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
private void assertParameterNotNull(Object parameterValue, String errorMessage) {
|
||||
if (parameterValue == null) throw new IllegalArgumentException(errorMessage);
|
||||
}
|
||||
|
||||
public static <X extends AmazonWebServiceRequest> X appendSingleObjectUserAgent(X request) {
|
||||
request.getRequestClientOptions().appendUserAgent(USER_AGENT);
|
||||
return request;
|
||||
}
|
||||
|
||||
private static final String USER_AGENT = TransferManager.class.getName() + "/" + VersionInfoUtils.getVersion();
|
||||
|
||||
private boolean isS3ObjectModifiedSincePause(final long lastModifiedTimeRecordedDuringResume,
|
||||
long lastModifiedTimeRecordedDuringPause) {
|
||||
return lastModifiedTimeRecordedDuringResume != lastModifiedTimeRecordedDuringPause;
|
||||
}
|
||||
|
||||
private final ScheduledExecutorService timedThreadPool = new ScheduledThreadPoolExecutor(1, daemonThreadFactory);
|
||||
|
||||
private static final ThreadFactory daemonThreadFactory = new ThreadFactory() {
|
||||
final AtomicInteger threadCount = new AtomicInteger( 0 );
|
||||
public Thread newThread(Runnable r) {
|
||||
int threadNumber = threadCount.incrementAndGet();
|
||||
Thread thread = new Thread(r);
|
||||
thread.setDaemon(true);
|
||||
thread.setName("S3TransferManagerTimedThread-" + threadNumber);
|
||||
return thread;
|
||||
}
|
||||
};
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
shutdownThreadPools();
|
||||
}
|
||||
|
||||
private void shutdownThreadPools() {
|
||||
if (shutDownThreadPools) {
|
||||
executorService.shutdown();
|
||||
timedThreadPool.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -4,7 +4,7 @@
|
||||
<groupId>org.talend</groupId>
|
||||
<artifactId>talend-httputil</artifactId>
|
||||
<name>talend-httputil</name>
|
||||
<version>1.0.5</version>
|
||||
<version>1.0.6</version>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
@@ -20,7 +20,7 @@
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>2.10.1</version>
|
||||
<version>2.11.4</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talendExcel-1.5-20200825</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>talendExcel</artifactId>
|
||||
<version>1.9-20201223</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talendExcel</name>
|
||||
@@ -43,47 +43,30 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi</artifactId>
|
||||
<version>4.1.0</version>
|
||||
<version>4.1.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-scratchpad</artifactId>
|
||||
<version>4.1.0</version>
|
||||
<version>4.1.2</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml</artifactId>
|
||||
<version>4.1.0</version>
|
||||
<version>4.1.2</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml-schemas -->
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml-schemas</artifactId>
|
||||
<version>4.1.0</version>
|
||||
<version>4.1.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.geronimo.specs/geronimo-stax-api_1.0_spec -->
|
||||
<dependency>
|
||||
<groupId>org.apache.geronimo.specs</groupId>
|
||||
<artifactId>geronimo-stax-api_1.0_spec</artifactId>
|
||||
<version>1.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dom4j</groupId>
|
||||
<artifactId>dom4j</artifactId>
|
||||
<version>2.1.3</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.xmlbeans/xmlbeans -->
|
||||
<dependency>
|
||||
<groupId>org.apache.xmlbeans</groupId>
|
||||
<artifactId>xmlbeans</artifactId>
|
||||
<version>3.1.0</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
|
||||
@@ -35,12 +35,12 @@ public class ExcelTool {
|
||||
|
||||
private Workbook wb = null;
|
||||
|
||||
private Workbook preWb = null;
|
||||
|
||||
private String sheetName = null;
|
||||
|
||||
private Sheet sheet = null;
|
||||
|
||||
private Workbook preWb = null;
|
||||
|
||||
private Sheet preSheet = null;
|
||||
|
||||
private Row curRow = null;
|
||||
@@ -80,6 +80,8 @@ public class ExcelTool {
|
||||
private boolean isTrackAllColumns = false;
|
||||
|
||||
private String password = null;
|
||||
|
||||
private Map<CellStyle, CellStyle> existedOriginToClone;
|
||||
|
||||
public ExcelTool() {
|
||||
cellStylesMapping = new HashMap<>();
|
||||
@@ -161,12 +163,8 @@ public class ExcelTool {
|
||||
}
|
||||
|
||||
private void appendActionForFile(String fileName) throws Exception {
|
||||
if (password == null) {
|
||||
InputStream inp = new FileInputStream(fileName);
|
||||
wb = WorkbookFactory.create(inp);
|
||||
} else {
|
||||
wb = readEncryptedFile(fileName);
|
||||
}
|
||||
InputStream inp = new FileInputStream(fileName);
|
||||
wb = WorkbookFactory.create(inp, password);
|
||||
sheet = wb.getSheet(sheetName);
|
||||
if (sheet != null) {
|
||||
if (appendSheet) {
|
||||
@@ -191,27 +189,10 @@ public class ExcelTool {
|
||||
}
|
||||
|
||||
private void initPreXlsx(String fileName) throws Exception {
|
||||
if(password == null) {
|
||||
InputStream preIns = new FileInputStream(fileName);
|
||||
preWb = WorkbookFactory.create(preIns);
|
||||
} else {
|
||||
preWb = readEncryptedFile(fileName);
|
||||
}
|
||||
InputStream preIns = new FileInputStream(fileName);
|
||||
preWb = WorkbookFactory.create(preIns, password);
|
||||
preSheet = preWb.getSheet(sheetName);
|
||||
}
|
||||
|
||||
private Workbook readEncryptedFile(String fileName)
|
||||
throws IOException, GeneralSecurityException {
|
||||
InputStream inp = new FileInputStream(fileName);
|
||||
POIFSFileSystem fs = new POIFSFileSystem(inp);
|
||||
EncryptionInfo info = new EncryptionInfo(fs);
|
||||
Decryptor decryptor = Decryptor.getInstance(info);
|
||||
if (!decryptor.verifyPassword(password)) {
|
||||
throw new GeneralSecurityException("Error: Incorrect password!");
|
||||
}
|
||||
InputStream dataStream = decryptor.getDataStream(fs);
|
||||
return WorkbookFactory.create(dataStream);
|
||||
}
|
||||
|
||||
public void setFont(String fontName) {
|
||||
if (StringUtils.isNotEmpty(fontName)) {
|
||||
@@ -222,11 +203,7 @@ public class ExcelTool {
|
||||
|
||||
public void addRow() {
|
||||
if (isAbsY && keepCellFormat) {
|
||||
if (preSheet != null) {
|
||||
preRow = preSheet.getRow(curY);
|
||||
} else {
|
||||
preRow = null;
|
||||
}
|
||||
preRow = (preSheet != null) ? preSheet.getRow(curY) : null;
|
||||
}
|
||||
curRow = sheet.getRow(curY);
|
||||
if (curRow == null) {
|
||||
@@ -244,11 +221,7 @@ public class ExcelTool {
|
||||
|
||||
private void addCell() {
|
||||
if (isAbsY && keepCellFormat) {
|
||||
if (preRow != null) {
|
||||
preCell = preRow.getCell(startX + xOffset);
|
||||
} else {
|
||||
preCell = null;
|
||||
}
|
||||
preCell = (preRow != null) ? preRow.getCell(startX + xOffset) : null;
|
||||
}
|
||||
curCell = curRow.createCell(startX + xOffset);
|
||||
xOffset++;
|
||||
@@ -267,10 +240,8 @@ public class ExcelTool {
|
||||
cellStylesMapping.put("normal", style);
|
||||
return style;
|
||||
}
|
||||
} else {
|
||||
return preCellStyle;
|
||||
}
|
||||
|
||||
return preCellStyle;
|
||||
}
|
||||
|
||||
private CellStyle getDateCellStyle(String pattern) {
|
||||
@@ -289,13 +260,15 @@ public class ExcelTool {
|
||||
cellStylesMapping.put(pattern, style);
|
||||
return style;
|
||||
}
|
||||
} else {
|
||||
return preCellStyle;
|
||||
}
|
||||
return preCellStyle;
|
||||
}
|
||||
|
||||
private CellStyle getPreCellStyle() {
|
||||
if (preSheet != null && isAbsY && keepCellFormat) {
|
||||
if(existedOriginToClone==null) {
|
||||
existedOriginToClone = new HashMap<>();
|
||||
}
|
||||
CellStyle preCellStyle;
|
||||
if (preCell == null) {
|
||||
preCellStyle = preSheet.getColumnStyle(curCell.getColumnIndex());
|
||||
@@ -303,14 +276,17 @@ public class ExcelTool {
|
||||
preCellStyle = preCell.getCellStyle();
|
||||
}
|
||||
|
||||
CellStyle targetCellStyle = wb.createCellStyle();
|
||||
targetCellStyle.cloneStyleFrom(preCellStyle);
|
||||
CellStyle targetCellStyle = existedOriginToClone.get(preCellStyle);
|
||||
if(targetCellStyle==null) {
|
||||
targetCellStyle = wb.createCellStyle();
|
||||
targetCellStyle.cloneStyleFrom(preCellStyle);
|
||||
existedOriginToClone.put(preCellStyle, targetCellStyle);
|
||||
}
|
||||
|
||||
return targetCellStyle;
|
||||
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public void addCellValue(boolean booleanValue) {
|
||||
@@ -364,7 +340,13 @@ public class ExcelTool {
|
||||
try {
|
||||
wb.write(outputStream);
|
||||
wb.close();
|
||||
if(preWb != null){
|
||||
preWb.close();
|
||||
}
|
||||
} finally {
|
||||
if(existedOriginToClone!=null) {
|
||||
existedOriginToClone = null;
|
||||
}
|
||||
if (outputStream != null) {
|
||||
outputStream.close();
|
||||
}
|
||||
@@ -382,13 +364,11 @@ public class ExcelTool {
|
||||
if (appendWorkbook && appendSheet && recalculateFormula) {
|
||||
evaluateFormulaCell();
|
||||
}
|
||||
FileOutputStream fileOutput = new FileOutputStream(fileName);
|
||||
POIFSFileSystem fs = null;
|
||||
try {
|
||||
try (FileOutputStream fileOutput = new FileOutputStream(fileName);
|
||||
POIFSFileSystem fs = new POIFSFileSystem()) {
|
||||
if (password == null) {
|
||||
wb.write(fileOutput);
|
||||
} else {
|
||||
fs = new POIFSFileSystem();
|
||||
Encryptor encryptor = new EncryptionInfo(EncryptionMode.agile).getEncryptor();
|
||||
encryptor.confirmPassword(password);
|
||||
OutputStream encryptedDataStream = encryptor.getDataStream(fs);
|
||||
@@ -397,10 +377,12 @@ public class ExcelTool {
|
||||
fs.writeFilesystem(fileOutput);
|
||||
}
|
||||
} finally {
|
||||
if(existedOriginToClone!=null) {
|
||||
existedOriginToClone = null;
|
||||
}
|
||||
wb.close();
|
||||
fileOutput.close();
|
||||
if (fs != null) {
|
||||
fs.close();
|
||||
if(preWb != null){
|
||||
preWb.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -411,8 +393,8 @@ public class ExcelTool {
|
||||
sheet = wb.getSheetAt(sheetNum);
|
||||
for (Row r : sheet) {
|
||||
for (Cell c : r) {
|
||||
if (c.getCellTypeEnum() == CellType.FORMULA) {
|
||||
evaluator.evaluateFormulaCellEnum(c);
|
||||
if (c.getCellType() == CellType.FORMULA) {
|
||||
evaluator.evaluateFormulaCell(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,61 +1,61 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talendMsgMailUtil-1.1-20191012</artifactId>
|
||||
<name>talendMsgMailUtil</name>
|
||||
<version>6.0.0</version>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>talendMsgMailUtil</artifactId>
|
||||
<version>1.2-20200923</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<java.source.version>1.5</java.source.version>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
</snapshotRepository>
|
||||
<repository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<java.source.version>1.8</java.source.version>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
</snapshotRepository>
|
||||
<repository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.25</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.25</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi</artifactId>
|
||||
<version>4.1.0</version>
|
||||
<version>4.1.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-scratchpad</artifactId>
|
||||
<version>4.1.0</version>
|
||||
<version>4.1.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.13</version>
|
||||
<version>1.2.17</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
@@ -65,19 +65,19 @@
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>${java.source.version}</source>
|
||||
<target>${java.source.version}</target>
|
||||
<showDeprecation>true</showDeprecation>
|
||||
<showWarnings>true</showWarnings>
|
||||
<compilerArgument>-XDignore.symbol.file</compilerArgument>
|
||||
<fork>true</fork>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>${java.source.version}</source>
|
||||
<target>${java.source.version}</target>
|
||||
<showDeprecation>true</showDeprecation>
|
||||
<showWarnings>true</showWarnings>
|
||||
<compilerArgument>-XDignore.symbol.file</compilerArgument>
|
||||
<fork>true</fork>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@@ -7,10 +7,6 @@ import java.io.OutputStream;
|
||||
|
||||
import org.apache.poi.hsmf.MAPIMessage;
|
||||
import org.apache.poi.hsmf.datatypes.AttachmentChunks;
|
||||
import org.apache.poi.hsmf.datatypes.Chunk;
|
||||
import org.apache.poi.hsmf.datatypes.MAPIProperty;
|
||||
import org.apache.poi.hsmf.datatypes.StringChunk;
|
||||
import org.apache.poi.hsmf.datatypes.Types;
|
||||
import org.apache.poi.hsmf.exceptions.ChunkNotFoundException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -36,12 +32,12 @@ public class MsgMailUtil {
|
||||
|
||||
public MsgMailUtil(String fileName, String outAttachmentPath)
|
||||
throws IOException {
|
||||
msg = new MAPIMessage(fileName);
|
||||
this.msg = new MAPIMessage(fileName);
|
||||
this.outAttachmentPath = outAttachmentPath;
|
||||
}
|
||||
|
||||
public void activeLog(String logger_name, String position) {
|
||||
this.log = LoggerFactory.getLogger(logger_name);
|
||||
public void activeLog(String loggerName, String position) {
|
||||
this.log = LoggerFactory.getLogger(loggerName);
|
||||
this.position = position;
|
||||
}
|
||||
|
||||
@@ -70,20 +66,14 @@ public class MsgMailUtil {
|
||||
}
|
||||
|
||||
File attachedFile = new File(dir, fileName);
|
||||
OutputStream fileOut = null;
|
||||
try {
|
||||
try(OutputStream fileOut = new FileOutputStream(attachedFile)) {
|
||||
processLog(Level.INFO, "Exporting attachment file :" + fileName);
|
||||
processLog(Level.INFO,
|
||||
"File location:" + attachedFile.getAbsolutePath());
|
||||
|
||||
fileOut = new FileOutputStream(attachedFile);
|
||||
fileOut.write(attachment.getEmbeddedAttachmentObject());
|
||||
|
||||
processLog(Level.INFO, "Export successfully");
|
||||
} finally {
|
||||
if (fileOut != null) {
|
||||
fileOut.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,9 +2,21 @@
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>talendzip</artifactId>
|
||||
<version>1.0-20190917</version>
|
||||
<version>1.1-20201120</version>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<source>8</source>
|
||||
<target>8</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
|
||||
@@ -2,6 +2,7 @@ package com.talend.compress.zip;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import net.lingala.zip4j.core.ZipFile;
|
||||
import net.lingala.zip4j.model.FileHeader;
|
||||
@@ -44,6 +45,8 @@ public class Unzip {
|
||||
this.useZip4jDecryption = useZip4jDecryption;
|
||||
}
|
||||
|
||||
public void setEncording(String encording){this.encording = encording;}
|
||||
|
||||
private boolean needPassword = false;
|
||||
private boolean useZip4jDecryption = false;
|
||||
|
||||
@@ -55,6 +58,8 @@ public class Unzip {
|
||||
|
||||
private String sourceZip;
|
||||
private String targetDir;
|
||||
private String encording;
|
||||
|
||||
|
||||
public Unzip(String sourceZip, String targetDir) {
|
||||
this.sourceZip = sourceZip;
|
||||
@@ -92,6 +97,9 @@ public class Unzip {
|
||||
}
|
||||
|
||||
ZipFile zipFile = new ZipFile(sourceZip);
|
||||
if(encording != null){
|
||||
zipFile.setFileNameCharset(encording);
|
||||
}
|
||||
|
||||
if (checkArchive) {
|
||||
if (!zipFile.isValidZipFile()) {
|
||||
@@ -152,9 +160,8 @@ public class Unzip {
|
||||
is = new javax.crypto.CipherInputStream(is,
|
||||
org.talend.archive.IntegrityUtil.createCipher(
|
||||
javax.crypto.Cipher.DECRYPT_MODE, password));
|
||||
|
||||
org.apache.commons.compress.archivers.zip.ZipArchiveInputStream input = new org.apache.commons.compress.archivers.zip.ZipArchiveInputStream(
|
||||
new java.io.BufferedInputStream(is));
|
||||
new java.io.BufferedInputStream(is),Optional.ofNullable(encording).orElse("UTF8"));
|
||||
org.apache.commons.compress.archivers.zip.ZipArchiveEntry entry;
|
||||
|
||||
while ((entry = input.getNextZipEntry()) != null) {
|
||||
@@ -190,7 +197,7 @@ public class Unzip {
|
||||
org.apache.commons.compress.archivers.zip.ZipFile zip = null;
|
||||
try {
|
||||
zip = new org.apache.commons.compress.archivers.zip.ZipFile(
|
||||
sourceZip);
|
||||
sourceZip,Optional.ofNullable(encording).orElse("UTF8"));
|
||||
java.util.Enumeration enuFiles = zip.getEntries();
|
||||
java.io.InputStream is = null;
|
||||
|
||||
|
||||
@@ -118,7 +118,7 @@
|
||||
<IMPORT NAME="Driver-JACKCESS" MODULE="jackcess-2.1.12.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess/2.1.12" REQUIRED="true"/>
|
||||
|
||||
<IMPORT NAME="jackcess-encrypt-2.1.4" MODULE="jackcess-encrypt-2.1.4.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess-encrypt/2.1.4" REQUIRED="true"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.60" MODULE="bcprov-jdk15on-1.60.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.60" REQUIRED="true"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.68" MODULE="bcprov-jdk15on-1.68.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.68" REQUIRED="true"/>
|
||||
<IMPORT NAME="talend-ucanaccess-utils-1.0.0" MODULE="talend-ucanaccess-utils-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-ucanaccess-utils-1.0.0/6.4.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.access/lib/talend-ucanaccess-utils-1.0.0.jar" REQUIRED="true"/>
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED="true"/>
|
||||
|
||||
@@ -121,7 +121,7 @@
|
||||
<IMPORT NAME="Driver-JACKCESS" MODULE="jackcess-2.1.12.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess/2.1.12" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="jackcess-encrypt-2.1.4" MODULE="jackcess-encrypt-2.1.4.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess-encrypt/2.1.4" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.60" MODULE="bcprov-jdk15on-1.60.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.60" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.68" MODULE="bcprov-jdk15on-1.68.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.68" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="talend-ucanaccess-utils-1.0.0" MODULE="talend-ucanaccess-utils-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-ucanaccess-utils-1.0.0/6.4.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.access/lib/talend-ucanaccess-utils-1.0.0.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
@@ -207,7 +207,7 @@
|
||||
<IMPORT NAME="Driver-JACKCESS" MODULE="jackcess-2.1.12.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess/2.1.12" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="jackcess-encrypt-2.1.4" MODULE="jackcess-encrypt-2.1.4.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess-encrypt/2.1.4" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.60" MODULE="bcprov-jdk15on-1.60.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.60" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.68" MODULE="bcprov-jdk15on-1.68.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.68" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="talend-ucanaccess-utils-1.0.0" MODULE="talend-ucanaccess-utils-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-ucanaccess-utils-1.0.0/6.4.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.access/lib/talend-ucanaccess-utils-1.0.0.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
@@ -181,7 +181,7 @@
|
||||
<IMPORT NAME="Driver-JACKCESS" MODULE="jackcess-2.1.12.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess/2.1.12" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="jackcess-encrypt-2.1.4" MODULE="jackcess-encrypt-2.1.4.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess-encrypt/2.1.4" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.60" MODULE="bcprov-jdk15on-1.60.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.60" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.68" MODULE="bcprov-jdk15on-1.68.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.68" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="talend-ucanaccess-utils-1.0.0" MODULE="talend-ucanaccess-utils-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-ucanaccess-utils-1.0.0/6.4.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.access/lib/talend-ucanaccess-utils-1.0.0.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
@@ -231,14 +231,14 @@
|
||||
<IMPORT NAME="aws-java-sdk-1.11.848.jar" MODULE="aws-java-sdk-1.11.848.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.848"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
<IMPORT NAME="jackson-core-2.11.4.jar" MODULE="jackson-core-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind-2.10.1.jar" MODULE="jackson-databind-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
<IMPORT NAME="jackson-databind-2.11.4.jar" MODULE="jackson-databind-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations-2.10.1.jar" MODULE="jackson-annotations-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
<IMPORT NAME="jackson-annotations-2.11.4.jar" MODULE="jackson-annotations-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
|
||||
@@ -558,14 +558,14 @@
|
||||
<IMPORT NAME="aws-java-sdk-1.11.848.jar" MODULE="aws-java-sdk-1.11.848.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.848"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
<IMPORT NAME="jackson-core-2.11.4.jar" MODULE="jackson-core-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind-2.10.1.jar" MODULE="jackson-databind-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
<IMPORT NAME="jackson-databind-2.11.4.jar" MODULE="jackson-databind-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations-2.10.1.jar" MODULE="jackson-annotations-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
<IMPORT NAME="jackson-annotations-2.11.4.jar" MODULE="jackson-annotations-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
|
||||
@@ -288,14 +288,14 @@
|
||||
<IMPORT NAME="aws-java-sdk-1.11.848.jar" MODULE="aws-java-sdk-1.11.848.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.848"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
<IMPORT NAME="jackson-core-2.11.4.jar" MODULE="jackson-core-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind-2.10.1.jar" MODULE="jackson-databind-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
<IMPORT NAME="jackson-databind-2.11.4.jar" MODULE="jackson-databind-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations-2.10.1.jar" MODULE="jackson-annotations-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
<IMPORT NAME="jackson-annotations-2.11.4.jar" MODULE="jackson-annotations-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
|
||||
@@ -54,11 +54,11 @@ for(IConnection conn : outgoingConns) {
|
||||
<%
|
||||
}
|
||||
}
|
||||
log4jCodeGenerateUtil.query(node);
|
||||
%>
|
||||
query_<%=cid %> = <%=dbquery%>;
|
||||
whetherReject_<%=cid%> = false;
|
||||
<%
|
||||
log4jCodeGenerateUtil.query(node, "query_" + cid);
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
if ((metadatas!=null)&&(metadatas.size()>0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
@@ -131,7 +131,7 @@ try {
|
||||
<%
|
||||
}
|
||||
}
|
||||
log4jCodeGenerateUtil.logInfo(node,"info",cid+" - Execute the query: '\" + "+dbquery +" + \"' has finished.");
|
||||
log4jCodeGenerateUtil.logInfo(node,"info",cid+" - Execute the query: '\" + query_" + cid + " + \"' has finished.");
|
||||
%>
|
||||
<% //feature 0010425
|
||||
if(usePrepareStatement){
|
||||
|
||||
@@ -348,14 +348,14 @@
|
||||
<IMPORT NAME="aws-java-sdk-1.11.848.jar" MODULE="aws-java-sdk-1.11.848.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.848"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
<IMPORT NAME="jackson-core-2.11.4.jar" MODULE="jackson-core-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind-2.10.1.jar" MODULE="jackson-databind-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
<IMPORT NAME="jackson-databind-2.11.4.jar" MODULE="jackson-databind-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations-2.10.1.jar" MODULE="jackson-annotations-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
<IMPORT NAME="jackson-annotations-2.11.4.jar" MODULE="jackson-annotations-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
|
||||
@@ -149,7 +149,7 @@
|
||||
<IMPORTS>
|
||||
<IMPORT
|
||||
NAME="Talen File Enhanced"
|
||||
MODULE="talend_file_enhanced-1.0.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.0.jar"
|
||||
MODULE="talend_file_enhanced-1.1.jar" MVN="mvn:org.talend.components.lib/talend_file_enhanced/1.1" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_file_enhanced-1.1.jar"
|
||||
REQUIRED="true"
|
||||
/>
|
||||
</IMPORTS>
|
||||
|
||||
@@ -200,7 +200,7 @@
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="DATE_PATTERN" GROUP="LOAD_DETAILS" FIELD="TEXT" NUM_ROW="15" SHOW_IF="isShow[(DATE_FORMAT] AND (DATE_FORMAT == 'true')">
|
||||
<PARAMETER NAME="DATE_PATTERN" GROUP="LOAD_DETAILS" FIELD="TEXT" NUM_ROW="15" SHOW_IF="isShow[DATE_FORMAT] AND (DATE_FORMAT == 'true')">
|
||||
<DEFAULT>"yyyy-MM-dd"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
<%@ jet
|
||||
imports="
|
||||
org.talend.core.model.process.INode
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
org.talend.core.model.process.INode
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
org.talend.designer.codegen.config.CodeGeneratorArgument
|
||||
org.talend.core.model.metadata.IMetadataTable
|
||||
org.talend.core.model.metadata.IMetadataColumn
|
||||
org.talend.core.model.metadata.IMetadataTable
|
||||
org.talend.core.model.metadata.IMetadataColumn
|
||||
org.talend.core.model.process.IConnection
|
||||
java.util.List
|
||||
"
|
||||
@@ -37,7 +37,7 @@
|
||||
boolean useCustomNullMarker = ElementParameterParser.getBooleanValue(node, "__USE_CUSTOM_NULL_MARKER__");
|
||||
String nullMarker = useCustomNullMarker ? ElementParameterParser.getValue(node, "__CUSTOM_NULL_MARKER__") : "\"\\\\N\"";
|
||||
String passwordFieldName = "";
|
||||
|
||||
|
||||
IConnection incomingConnection = null;
|
||||
if(node.getUniqueName().startsWith("tBigQueryOutput_")) {
|
||||
List< ? extends IConnection> conns = node.getIncomingConnections();
|
||||
@@ -49,7 +49,7 @@
|
||||
incomingConnection = virtConnection.getSource().getIncomingConnections().get(0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (authMode.equals("OAUTH")) {
|
||||
%>
|
||||
/* ----START-CREATING-CLIENT (OAuth 2.0)---- */
|
||||
@@ -58,22 +58,22 @@
|
||||
<%
|
||||
passwordFieldName = "__CLIENT_SECRET__";
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
final String CLIENT_SECRET_<%=cid%> = "{\"web\": {\"client_id\": \""+<%=clientId%>+"\",\"client_secret\": \"" +decryptedPassword_<%=cid%>+ "\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://accounts.google.com/o/oauth2/token\"}}";
|
||||
final String PROJECT_ID_<%=cid %> = <%=projectId %>;
|
||||
|
||||
|
||||
// Static variables for API scope, callback URI, and HTTP/JSON functions
|
||||
final List<String> SCOPES_<%=cid%> = java.util.Arrays.asList("https://www.googleapis.com/auth/bigquery");
|
||||
final String REDIRECT_URI_<%=cid%> = "urn:ietf:wg:oauth:2.0:oob";
|
||||
final com.google.api.client.http.HttpTransport TRANSPORT_<%=cid %> = new com.google.api.client.http.javanet.NetHttpTransport();
|
||||
final com.google.api.client.json.JsonFactory JSON_FACTORY_<%=cid %> = new com.google.api.client.json.jackson2.JacksonFactory();
|
||||
|
||||
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets clientSecrets_<%=cid%> = com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets.load(
|
||||
new com.google.api.client.json.jackson2.JacksonFactory(), new java.io.InputStreamReader(new java.io.ByteArrayInputStream(
|
||||
CLIENT_SECRET_<%=cid%>.getBytes())));
|
||||
|
||||
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow flow_<%=cid%> = null;
|
||||
com.google.api.services.bigquery.Bigquery bigqueryclient_<%=cid%> = null;
|
||||
long nb_line_<%=cid%> = 0;
|
||||
@@ -111,7 +111,7 @@
|
||||
%>
|
||||
}
|
||||
String storedRefreshToken_<%=cid%> = (String) properties_<%=cid%>.get("refreshtoken");
|
||||
|
||||
|
||||
// Check to see if the an existing refresh token was loaded.
|
||||
// If so, create a credential and call refreshToken() to get a new
|
||||
// access token.
|
||||
@@ -120,7 +120,7 @@
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleCredential credential_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2. GoogleCredential.Builder().setTransport(TRANSPORT_<%=cid%>)
|
||||
.setJsonFactory(JSON_FACTORY_<%=cid%>).setClientSecrets(clientSecrets_<%=cid%>)
|
||||
.build().setFromTokenResponse(new com.google.api.client.auth.oauth2.TokenResponse().setRefreshToken(storedRefreshToken_<%=cid%>));
|
||||
|
||||
|
||||
credential_<%=cid%>.refreshToken();
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
@@ -186,13 +186,13 @@
|
||||
if (outputStream_<%=cid%> != null) {
|
||||
outputStream_<%=cid%>.close();
|
||||
}
|
||||
|
||||
|
||||
bigqueryclient_<%=cid%> = new com.google.api.services.bigquery.Bigquery.Builder(new com.google.api.client.http.javanet.NetHttpTransport(),new com.google.api.client.json.jackson2.JacksonFactory(),credential_<%=cid%>).build();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ----END-CREATING-CLIENT (OAuth 2.0)---- */
|
||||
|
||||
|
||||
<%
|
||||
} else if (authMode.equals("SERVICEACCOUNT")) {
|
||||
%>
|
||||
@@ -213,9 +213,35 @@
|
||||
/* ----END-CREATING-CLIENT (Cloud API)---- */
|
||||
|
||||
long nb_line_<%=cid%> = 0;
|
||||
<%
|
||||
} else if (authMode.equals("TOKEN")) {
|
||||
if (ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) {%>
|
||||
final String decryptedAccessToken_<%=cid%> = routines.system.PasswordEncryptUtil.decryptPassword(<%=ElementParameterParser.getEncryptedValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<%} else {%>
|
||||
final String decryptedAccessToken_<%=cid%> = <%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>;
|
||||
<%}%>
|
||||
/* ----START-CREATING-CLIENT (OAuth based Token)---- */
|
||||
final String PROJECT_ID_<%=cid %> = <%=projectId %>;
|
||||
|
||||
com.google.api.client.auth.oauth2.Credential cred_<%=cid%> =
|
||||
new com.google.api.client.auth.oauth2.Credential(com.google.api.client.auth.oauth2.BearerToken.authorizationHeaderAccessMethod()).setFromTokenResponse(
|
||||
(new com.google.api.client.auth.oauth2.TokenResponse()).setAccessToken(decryptedAccessToken_<%=cid%>));
|
||||
|
||||
com.google.api.services.bigquery.Bigquery bigqueryclient_<%=cid%> = null;
|
||||
long nb_line_<%=cid%> = 0;
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Creating client.");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
bigqueryclient_<%=cid%> =
|
||||
new com.google.api.services.bigquery.Bigquery.Builder(new com.google.api.client.http.javanet.NetHttpTransport(), new com.google.api.client.json.jackson2.JacksonFactory(), cred_<%=cid%>).setApplicationName("Talend").build();
|
||||
/* ----END-CREATING-CLIENT (OAuth based Token)---- */
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\", \"OAUTH\" or \"TOKEN\", but it is " + authMode);
|
||||
}
|
||||
boolean bulkFileAlreadyExists = ElementParameterParser.getBooleanValue(node, "__BULK_FILE_ALREADY_EXIST__");
|
||||
String accessKey = ElementParameterParser.getValue(node, "__GS_ACCESS_KEY__");
|
||||
@@ -227,17 +253,17 @@
|
||||
|
||||
|
||||
if(!bulkFileAlreadyExists) {
|
||||
if (!useServiceAccountForConnection){
|
||||
if ("USER_ACCOUNT_HMAC".equals(authType)){
|
||||
%>
|
||||
/* ----START-UPLOADING-FILE WITH HMAC ACCOUNT---- */
|
||||
<%
|
||||
passwordFieldName = "__GS_SECRET_KEY__";
|
||||
%>
|
||||
|
||||
<%if (ElementParameterParser.canEncrypt(node, passwordFieldName)) {%>
|
||||
|
||||
<%if (ElementParameterParser.canEncrypt(node, passwordFieldName)) {%>
|
||||
final String decryptedPwd_<%=cid%> = routines.system.PasswordEncryptUtil.decryptPassword(<%=ElementParameterParser.getEncryptedValue(node, passwordFieldName)%>);
|
||||
<%} else {%>
|
||||
final String decryptedPwd_<%=cid%> = <%= ElementParameterParser.getValue(node, passwordFieldName)%>;
|
||||
final String decryptedPwd_<%=cid%> = <%= ElementParameterParser.getValue(node, passwordFieldName)%>;
|
||||
<%}%>
|
||||
|
||||
|
||||
@@ -262,33 +288,12 @@
|
||||
log.info("<%=cid%> - Upload Done.");
|
||||
<%
|
||||
}
|
||||
} else {
|
||||
String pathToServerAccoutKeyFile = ElementParameterParser.getValue(node, "__GS_SERVICE_ACCOUNT_KEY__");
|
||||
%>
|
||||
/* ----START-UPLOADING-FILE WITH SERVICE ACCOUNT---- */
|
||||
com.google.cloud.storage.Storage storage_<%=cid%> = null;
|
||||
com.google.auth.Credentials credential_<%=cid%> = null;
|
||||
try {
|
||||
credential_<%=cid%> = com.google.auth.oauth2.GoogleCredentials
|
||||
.fromStream(new java.io.FileInputStream(<%=pathToServerAccoutKeyFile%>));
|
||||
} catch (IOException e_<%=cid%>) {
|
||||
<%
|
||||
if (("true").equals(dieOnError)) {
|
||||
%>
|
||||
throw(e_<%=cid%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.error("<%=cid%> - Exception in component <%=cid%>.", e_<%=cid%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
com.google.cloud.storage.StorageOptions.Builder builder = com.google.cloud.storage.StorageOptions.newBuilder();
|
||||
storage_<%=cid%> = builder.setCredentials(credential_<%=cid%>).build().getService();
|
||||
} else {
|
||||
boolean useExistingConn = false;
|
||||
String gsProjectID = projectId;
|
||||
String connection = cid;
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/googleStorageConnection.javajet"%>
|
||||
|
||||
java.io.File file_<%=cid%> = new java.io.File(<%=localFilename%>);
|
||||
|
||||
@@ -302,6 +307,7 @@
|
||||
<%
|
||||
}
|
||||
%>
|
||||
/* ----END-UPLOADING-FILE---- */
|
||||
<%
|
||||
}
|
||||
}
|
||||
@@ -313,7 +319,7 @@
|
||||
int currIndex_<%=cid%> = 0;
|
||||
<%
|
||||
}
|
||||
if (authMode.equals("OAUTH")) {
|
||||
if (authMode.equals("OAUTH") || authMode.equals("TOKEN")) {
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Starting build a job.");
|
||||
@@ -323,12 +329,15 @@
|
||||
/* ----START-CREATING-JOB (OAuth 2.0)---- */
|
||||
com.google.api.services.bigquery.model.Job job_<%=cid%> = new com.google.api.services.bigquery.model.Job();
|
||||
job_<%=cid%>.setJobReference(new com.google.api.services.bigquery.model.JobReference().setProjectId(PROJECT_ID_<%=cid%>));
|
||||
|
||||
|
||||
com.google.api.services.bigquery.model.JobConfiguration config_<%=cid%> = new com.google.api.services.bigquery.model.JobConfiguration();
|
||||
com.google.api.services.bigquery.model.JobConfigurationLoad queryLoad_<%=cid%> = new com.google.api.services.bigquery.model.JobConfigurationLoad();
|
||||
|
||||
|
||||
<%if (dropTable) {%>
|
||||
try {
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
cred_<%=cid%>.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
bigqueryclient_<%=cid%>.tables().delete(PROJECT_ID_<%=cid%>, <%=dataset%>, <%=table%>).execute();
|
||||
} catch (com.google.api.client.googleapis.json.GoogleJsonResponseException e_<%=cid%>) {
|
||||
if (e_<%=cid%>.getDetails().getCode() != 404) {
|
||||
@@ -351,10 +360,13 @@
|
||||
}
|
||||
}
|
||||
<%}%>
|
||||
|
||||
|
||||
<%if (createTableIfNotExist) { %>
|
||||
|
||||
|
||||
try {
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
cred_<%=cid%>.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
com.google.api.services.bigquery.model.Table getTable = bigqueryclient_<%=cid%>.tables().get(PROJECT_ID_<%=cid%>, <%=dataset%>, <%=table%>).execute();
|
||||
queryLoad_<%=cid%>.setCreateDisposition("CREATE_NEVER");
|
||||
} catch (com.google.api.client.googleapis.json.GoogleJsonResponseException e_<%=cid%>){
|
||||
@@ -497,15 +509,17 @@
|
||||
destinationTable_<%=cid%>.setProjectId(PROJECT_ID_<%=cid%>);
|
||||
destinationTable_<%=cid%>.setDatasetId(<%=dataset%>);
|
||||
destinationTable_<%=cid%>.setTableId(<%=table%>);
|
||||
|
||||
|
||||
queryLoad_<%=cid%>.setDestinationTable(destinationTable_<%=cid%>);
|
||||
queryLoad_<%=cid%>.setSourceUris(java.util.Arrays.asList(<%=gsFile%>));
|
||||
queryLoad_<%=cid%>.setSkipLeadingRows(<%=gsFileHeader%>);
|
||||
queryLoad_<%=cid%>.setNullMarker(<%= nullMarker %>);
|
||||
config_<%=cid%>.setLoad(queryLoad_<%=cid%>);
|
||||
|
||||
|
||||
job_<%=cid%>.setConfiguration(config_<%=cid%>);
|
||||
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
cred_<%=cid%>.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
com.google.api.services.bigquery.Bigquery.Jobs.Insert insertReq_<%=cid%> = bigqueryclient_<%=cid%>.jobs().insert("", job_<%=cid%>);
|
||||
insertReq_<%=cid%>.setProjectId(PROJECT_ID_<%=cid%>);
|
||||
<%
|
||||
@@ -514,11 +528,17 @@
|
||||
log.info("<%=cid%> - Build a job successfully.");
|
||||
log.info("<%=cid%> - Starting load the job.");
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
System.out.println("Starting load job.");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
System.out.println("Starting load job.");
|
||||
com.google.api.services.bigquery.model.Job jobExec_<%=cid%> = null;
|
||||
try {
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
cred_<%=cid%>.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
jobExec_<%=cid%> = insertReq_<%=cid%>.execute();
|
||||
} catch (Exception ee_<%=cid%>) {
|
||||
<%
|
||||
@@ -540,9 +560,15 @@
|
||||
}
|
||||
if (jobExec_<%=cid%>.getStatus().getState().equals("RUNNING")
|
||||
|| jobExec_<%=cid%>.getStatus().getState().equals("PENDING")) {
|
||||
com.google.api.services.bigquery.model.Job pollJob_<%=cid%> = bigqueryclient_<%=cid%>.jobs().get(PROJECT_ID_<%=cid%>,jobExec_<%=cid%>.getJobReference().getJobId()).setLocation(jobExec_<%=cid%>.getJobReference().getLocation()).execute();
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
cred_<%=cid%>.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
com.google.api.services.bigquery.model.Job pollJob_<%=cid%> = bigqueryclient_<%=cid%>.jobs().get(PROJECT_ID_<%=cid%>,jobExec_<%=cid%>.getJobReference().getJobId()).setLocation(jobExec_<%=cid%>.getJobReference().getLocation()).execute();
|
||||
while (pollJob_<%=cid%>.getStatus().getState().equals("RUNNING") || pollJob_<%=cid%>.getStatus().getState().equals("PENDING")) {
|
||||
Thread.sleep(1000);
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
cred_<%=cid%>.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
pollJob_<%=cid%> = bigqueryclient_<%=cid%>.jobs().get(PROJECT_ID_<%=cid%>,jobExec_<%=cid%>.getJobReference().getJobId()).setLocation(jobExec_<%=cid%>.getJobReference().getLocation()).execute();
|
||||
System.out.println(String.format(
|
||||
"Waiting on job %s ... Current status: %s", jobExec_<%=cid%>
|
||||
@@ -556,14 +582,17 @@
|
||||
}
|
||||
%>
|
||||
}
|
||||
|
||||
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
cred_<%=cid%>.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
com.google.api.services.bigquery.model.Job doneJob_<%=cid%> = pollJob_<%=cid%>;
|
||||
|
||||
|
||||
if ((doneJob_<%=cid%>.getStatus() != null) && (doneJob_<%=cid%>.getStatus().getErrors() != null)) {
|
||||
status = "failure";
|
||||
throw new Exception(doneJob_<%=cid%>.getStatus().getErrors().toString());
|
||||
}
|
||||
|
||||
|
||||
System.out.println("Done: " + doneJob_<%=cid%>.toString());
|
||||
com.google.api.services.bigquery.model.JobStatistics jobStatistics_<%=cid%>= doneJob_<%=cid%>.getStatistics();
|
||||
if(jobStatistics_<%=cid%>!=null && jobStatistics_<%=cid%>.getLoad() != null){
|
||||
@@ -605,11 +634,11 @@
|
||||
/* ----START-CREATING-JOB (Cloud API)---- */
|
||||
com.google.cloud.bigquery.TableId tableId_<%=cid%> = com.google.cloud.bigquery.TableId.of(<%=projectId%>, <%=dataset%>, <%=table%>);
|
||||
com.google.cloud.bigquery.LoadJobConfiguration.Builder loadJobBuilder_<%=cid%> = com.google.cloud.bigquery.LoadJobConfiguration.newBuilder(tableId_<%=cid%>, <%=gsFile%>);
|
||||
|
||||
|
||||
boolean dropTable_<%=cid%> = <%=dropTable%>;
|
||||
|
||||
if (dropTable_<%=cid%> && bigquery_<%=cid%>.getTable(tableId_<%=cid%>) != null) {
|
||||
boolean deleted = bigquery_<%=cid%>.delete(tableId_<%=cid%>);
|
||||
boolean deleted = bigquery_<%=cid%>.delete(tableId_<%=cid%>);
|
||||
if (deleted) {
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
@@ -645,10 +674,10 @@
|
||||
} else if ("id_Short".equals(column.getTalendType()) || "id_Integer".equals(column.getTalendType()) || "id_Long".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.INTEGER";
|
||||
} else if ("id_BigDecimal".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.NUMERIC";
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.NUMERIC";
|
||||
} else if ("id_Boolean".equals(column.getTalendType())) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.BOOLEAN";
|
||||
} else if ("id_Date".equals(column.getTalendType())) {
|
||||
} else if ("id_Date".equals(column.getTalendType())) {
|
||||
String pattern = column.getPattern();
|
||||
if(pattern.length() == 12 || pattern.isEmpty() || "\"\"".equals(pattern)) {
|
||||
typeToGenerate = "com.google.cloud.bigquery.LegacySQLTypeName.DATE";
|
||||
@@ -660,11 +689,11 @@
|
||||
}
|
||||
String modeType = (!column.isNullable()) ? "REQUIRED" : "NULLABLE";
|
||||
%>
|
||||
|
||||
|
||||
com.google.cloud.bigquery.Field field_<%=i%> = com.google.cloud.bigquery.Field.newBuilder("<%=columnName%>", <%=typeToGenerate%>)
|
||||
.setMode(com.google.cloud.bigquery.Field.Mode.valueOf("<%=modeType%>"))
|
||||
.build();
|
||||
fields_<%=cid%>.add(field_<%=i%>);
|
||||
fields_<%=cid%>.add(field_<%=i%>);
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
@@ -753,32 +782,32 @@
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
|
||||
com.google.cloud.bigquery.Schema schema_<%=cid%> = com.google.cloud.bigquery.Schema.of(fields_<%=cid%>);
|
||||
com.google.cloud.bigquery.TableInfo tableInfo_<%=cid%> = com.google.cloud.bigquery.TableInfo.newBuilder(tableId_<%=cid%>, com.google.cloud.bigquery.StandardTableDefinition.of(schema_<%=cid%>)).build();
|
||||
com.google.cloud.bigquery.Table table_<%=cid%> = bigquery_<%=cid%>.create(tableInfo_<%=cid%>);
|
||||
loadJobBuilder_<%=cid%>.setSchema(schema_<%=cid%>);
|
||||
loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_IF_NEEDED);
|
||||
|
||||
|
||||
} else {
|
||||
loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_NEVER);
|
||||
}
|
||||
<%} else {%>
|
||||
loadJobBuilder_<%=cid%>.setCreateDisposition(com.google.cloud.bigquery.JobInfo.CreateDisposition.CREATE_NEVER);
|
||||
<%}%>
|
||||
|
||||
loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_<%=actionOnData%>);
|
||||
|
||||
loadJobBuilder_<%=cid%>.setWriteDisposition(com.google.cloud.bigquery.JobInfo.WriteDisposition.WRITE_<%=actionOnData%>);
|
||||
loadJobBuilder_<%=cid%>.setDestinationTable(tableId_<%=cid%>);
|
||||
com.google.cloud.bigquery.CsvOptions.Builder csvOptions_<%=cid%> = com.google.cloud.bigquery.CsvOptions.newBuilder();
|
||||
csvOptions_<%=cid%>.setAllowQuotedNewLines(true);
|
||||
csvOptions_<%=cid%>.setSkipLeadingRows(<%=gsFileHeader%>);
|
||||
|
||||
|
||||
<%if(setFieldDelimiter) {
|
||||
%>
|
||||
csvOptions_<%=cid%>.setFieldDelimiter(<%=fieldDelimiter%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
%>
|
||||
loadJobBuilder_<%=cid%>.setFormatOptions(csvOptions_<%=cid%>.build());
|
||||
loadJobBuilder_<%=cid%>.setNullMarker(<%= nullMarker %>);
|
||||
com.google.cloud.bigquery.Job job_<%=cid%> = bigquery_<%=cid%>.create(com.google.cloud.bigquery.JobInfo.of(loadJobBuilder_<%=cid%>.build()));
|
||||
@@ -795,10 +824,10 @@
|
||||
List<com.google.cloud.bigquery.BigQueryError> errorList = job_<%=cid%>.getStatus().getExecutionErrors();
|
||||
throw new RuntimeException("Job failed: " + errorList.get(errorList.size() - 1));
|
||||
}
|
||||
|
||||
|
||||
/* ----END-CREATING-JOB (Cloud API)---- */
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\", \"OAUTH\" or \"TOKEN\", but it is " + authMode);
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -39,8 +39,31 @@
|
||||
<ITEMS DEFAULT="SERVICEACCOUNT">
|
||||
<ITEM NAME="SERVICEACCOUNT" VALUE="SERVICEACCOUNT" />
|
||||
<ITEM NAME="OAUTH" VALUE="OAUTH" />
|
||||
<ITEM NAME="TOKEN" VALUE="TOKEN" />
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="ACCESS_TOKEN"
|
||||
FIELD="PASSWORD"
|
||||
NUM_ROW="10"
|
||||
REQUIRED="true"
|
||||
SHOW_IF="AUTH_MODE == 'TOKEN'"
|
||||
GROUP="AUTHENTICATION"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="ACCESS_TOKEN_WARNING"
|
||||
FIELD="LABEL"
|
||||
NUM_ROW="11"
|
||||
REQUIRED="false"
|
||||
GROUP="AUTHENTICATION"
|
||||
SHOW_IF="AUTH_MODE == 'TOKEN'"
|
||||
>
|
||||
<DEFAULT>*Note: If the Access Token Expire we won't be able to refresh it!"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="SERVICE_ACCOUNT_CREDENTIALS_FILE"
|
||||
@@ -144,6 +167,7 @@
|
||||
<ITEMS DEFAULT="GS_SERVICE_ACCOUNT">
|
||||
<ITEM NAME="USER_ACCOUNT_HMAC" VALUE="USER_ACCOUNT_HMAC"/>
|
||||
<ITEM NAME="GS_SERVICE_ACCOUNT" VALUE="GS_SERVICE_ACCOUNT"/>
|
||||
<ITEM NAME="TOKEN" VALUE="TOKEN" />
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
|
||||
@@ -177,6 +201,28 @@
|
||||
<DEFAULT>"__COMP_DEFAULT_FILE_DIR__/key.json"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="GS_ACCESS_TOKEN"
|
||||
FIELD="PASSWORD"
|
||||
NUM_ROW="60"
|
||||
REQUIRED_IF="(BULK_FILE_ALREADY_EXIST=='false') AND (AUTH_TYPE == 'TOKEN')"
|
||||
SHOW_IF="(BULK_FILE_ALREADY_EXIST=='false') AND (AUTH_TYPE == 'TOKEN')"
|
||||
GROUP="GS_CONF"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="GS_ACCESS_TOKEN_WARNING"
|
||||
FIELD="LABEL"
|
||||
NUM_ROW="61"
|
||||
REQUIRED="false"
|
||||
GROUP="GS_CONF"
|
||||
SHOW_IF="(BULK_FILE_ALREADY_EXIST=='false') AND (AUTH_TYPE == 'TOKEN')"
|
||||
>
|
||||
<DEFAULT>*Note: If the Access Token Expire we won't be able to refresh it!"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="GS_LOCAL_FILE" FIELD="FILE" NUM_ROW="63" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
@@ -233,47 +279,50 @@
|
||||
</ADVANCED_PARAMETERS>
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="google-api-client-1.25.0.jar" MODULE="google-api-client-1.25.0.jar" MVN="mvn:com.google.api-client/google-api-client/1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-services-oauth2-v2-rev151-1.25.0.jar" MODULE="google-api-services-oauth2-v2-rev151-1.25.0.jar" MVN="mvn:com.google.apis/google-api-services-oauth2/v2-rev151-1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-services-bigquery-v2-rev454-1.25.0.jar" MODULE="google-api-services-bigquery-v2-rev454-1.25.0.jar" MVN="mvn:com.google.apis/google-api-services-bigquery/v2-rev454-1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-1.25.0.jar" MODULE="google-http-client-1.25.0.jar" MVN="mvn:com.google.http-client/google-http-client/1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-oauth-client-1.25.0.jar" MODULE="google-oauth-client-1.25.0.jar" MVN="mvn:com.google.oauth-client/google-oauth-client/1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-jackson2-1.25.0.jar" MODULE="google-http-client-jackson2-1.25.0.jar" MVN="mvn:com.google.http-client/google-http-client-jackson2/1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-client-1.31.1.jar" MODULE="google-api-client-1.31.1.jar" MVN="mvn:com.google.api-client/google-api-client/1.31.1" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-services-oauth2-v2-rev20200213-1.31.0.jar" MODULE="google-api-services-oauth2-v2-rev20200213-1.31.0.jar" MVN="mvn:com.google.apis/google-api-services-oauth2/v2-rev20200213-1.31.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-services-bigquery-v2-rev20201030-1.31.0.jar" MODULE="google-api-services-bigquery-v2-rev20201030-1.31.0.jar" MVN="mvn:com.google.apis/google-api-services-bigquery/v2-rev20201030-1.31.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-1.38.0.jar" MODULE="google-http-client-1.38.0.jar" MVN="mvn:com.google.http-client/google-http-client/1.38.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-oauth-client-1.31.0.jar" MODULE="google-oauth-client-1.31.0.jar" MVN="mvn:com.google.oauth-client/google-oauth-client/1.31.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-jackson2-1.38.0.jar" MODULE="google-http-client-jackson2-1.38.0.jar" MVN="mvn:com.google.http-client/google-http-client-jackson2/1.38.0" REQUIRED="true" />
|
||||
<IMPORT NAME="guava-20.0.jar" MODULE="guava-20.0.jar" MVN="mvn:com.google.guava/guava/20.0" REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1" REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.11.4.jar" MODULE="jackson-core-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4" REQUIRED="true" />
|
||||
<!-- REQUIRED FOR GOOGLE STORAGE -->
|
||||
<IMPORT NAME="jets3t-0.9.1" MODULE="jets3t-0.9.1.jar" MVN="mvn:org.talend.libraries/jets3t-0.9.1/6.0.0" REQUIRED="true" />
|
||||
<IMPORT NAME="commons-logging-1.2" MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2" REQUIRED="true" />
|
||||
<IMPORT NAME="httpclient-4.5.12" MODULE="httpclient-4.5.12.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.12" REQUIRED="true" />
|
||||
<IMPORT NAME="httpcore-4.4.13" MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13" REQUIRED="true" />
|
||||
<IMPORT NAME="commons-codec-1.14" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED="true" />
|
||||
<IMPORT NAME="google-cloud-bigquery-1.60.0.jar" MODULE="google-cloud-bigquery-1.60.0.jar" MVN="mvn:com.google.cloud/google-cloud-bigquery/1.60.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="google-http-client-jackson-1.25.0.jar" MODULE="google-http-client-jackson-1.25.0.jar" MVN="mvn:com.google.http-client/google-http-client-jackson/1.25.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="threetenbp-1.3.3.jar" MODULE="threetenbp-1.3.3.jar" MVN="mvn:org.threeten/threetenbp/1.3.3" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="google-auth-library-credentials-0.20.0.jar" MODULE="google-auth-library-credentials-0.20.0.jar" MVN="mvn:com.google.auth/google-auth-library-credentials/0.20.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="gax-httpjson-0.44.0.jar" MODULE="gax-httpjson-0.44.0.jar" MVN="mvn:com.google.api/gax-httpjson/0.44.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="jackson-core-asl-1.9.13.jar" MODULE="jackson-core-asl-1.9.13.jar" MVN="mvn:org.codehaus.jackson/jackson-core-asl/1.9.13" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="google-auth-library-oauth2-http-0.20.0.jar" MODULE="google-auth-library-oauth2-http-0.20.0.jar" MVN="mvn:com.google.auth/google-auth-library-oauth2-http/0.20.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="google-cloud-core-1.93.4.jar" MODULE="google-cloud-core-1.93.4.jar" MVN="mvn:com.google.cloud/google-cloud-core/1.93.4" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="google-cloud-core-http-1.32.0.jar" MODULE="google-cloud-core-http-1.32.0.jar" MVN="mvn:com.google.cloud/google-cloud-core-http/1.32.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="gax-1.27.0.jar" MODULE="gax-1.27.0.jar" MVN="mvn:com.google.api/gax/1.27.0" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="google-http-client-appengine-1.25.0.jar" MODULE="google-http-client-appengine-1.25.0.jar" MVN="mvn:com.google.http-client/google-http-client-appengine/1.25.0" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="api-common-1.6.0.jar" MODULE="api-common-1.6.0.jar" MVN="mvn:com.google.api/api-common/1.6.0" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="google-cloud-bigquery-1.126.3.jar" MODULE="google-cloud-bigquery-1.126.3.jar" MVN="mvn:com.google.cloud/google-cloud-bigquery/1.126.3" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
|
||||
<!-- TODO remove it -->
|
||||
<IMPORT NAME="google-http-client-jackson-1.25.0.jar" MODULE="google-http-client-jackson-1.25.0.jar" MVN="mvn:com.google.http-client/google-http-client-jackson/1.25.0" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
|
||||
<IMPORT NAME="threetenbp-1.3.3.jar" MODULE="threetenbp-1.3.3.jar" MVN="mvn:org.threeten/threetenbp/1.3.3" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="google-auth-library-credentials-0.20.0.jar" MODULE="google-auth-library-credentials-0.20.0.jar" MVN="mvn:com.google.auth/google-auth-library-credentials/0.20.0" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="gax-httpjson-0.44.0.jar" MODULE="gax-httpjson-0.44.0.jar" MVN="mvn:com.google.api/gax-httpjson/0.44.0" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="jackson-core-asl-1.9.13.jar" MODULE="jackson-core-asl-1.9.13.jar" MVN="mvn:org.codehaus.jackson/jackson-core-asl/1.9.13" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="google-auth-library-oauth2-http-0.20.0.jar" MODULE="google-auth-library-oauth2-http-0.20.0.jar" MVN="mvn:com.google.auth/google-auth-library-oauth2-http/0.20.0" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="google-cloud-core-1.93.4.jar" MODULE="google-cloud-core-1.93.4.jar" MVN="mvn:com.google.cloud/google-cloud-core/1.93.4" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="google-cloud-core-http-1.32.0.jar" MODULE="google-cloud-core-http-1.32.0.jar" MVN="mvn:com.google.cloud/google-cloud-core-http/1.32.0" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="gax-1.27.0.jar" MODULE="gax-1.27.0.jar" MVN="mvn:com.google.api/gax/1.27.0" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="google-http-client-appengine-1.38.0.jar" MODULE="google-http-client-appengine-1.38.0.jar" MVN="mvn:com.google.http-client/google-http-client-appengine/1.38.0" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="api-common-1.6.0.jar" MODULE="api-common-1.6.0.jar" MVN="mvn:com.google.api/api-common/1.6.0" REQUIRED_IF="(AUTH_MODE == 'SERVICEACCOUNT') OR (AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
|
||||
<IMPORT NAME="google-cloud-storage-1.104.0" MODULE="google-cloud-storage-1.104.0.jar" MVN="mvn:com.google.cloud/google-cloud-storage/1.104.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="google-cloud-storage-1.104.0" MODULE="google-cloud-storage-1.104.0.jar" MVN="mvn:com.google.cloud/google-cloud-storage/1.104.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<!-- Transitive dependencies of google-cloud-storage -->
|
||||
<IMPORT NAME="google-api-services-storage-v1-rev20191011-1.30.3" MODULE="google-api-services-storage-v1-rev20191011-1.30.3.jar" MVN="mvn:com.google.apis/google-api-services-storage/v1-rev20191011-1.30.3" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="gson-2.8.6" MODULE="gson-2.8.6.jar" MVN="mvn:com.google.code.gson/gson/2.8.6" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="javax.annotation-api" MODULE="javax.annotation-api-1.3.jar" MVN="mvn:javax.annotation/javax.annotation-api/1.3" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="protobuf-java-2.5.0" MODULE="protobuf-java-2.5.0.jar" MVN="mvn:com.google.protobuf/protobuf-java/2.5.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="error_prone_annotation-2.1.3" MODULE="error_prone_annotation-2.1.3.jar" MVN="mvn:com.google.errorprone/error_prone_annotations/2.1.3" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="opencensus-api-0.21.0" MODULE="opencensus-api-0.21.0.jar" MVN="mvn:io.opencensus/opencensus-api/0.21.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="opencensus-contrib-http-util-0.21.0" MODULE="opencensus-contrib-http-util-0.21.0.jar" MVN="mvn:io.opencensus/opencensus-contrib-http-util/0.21.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="grpc-context-1.19.0" MODULE="grpc-context-1.19.0.jar" MVN="mvn:io.grpc/grpc-context/1.19.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="proto-google-common-protos-1.17.0" MODULE="proto-google-common-protos-1.17.0.jar" MVN="mvn:com.google.api.grpc/proto-google-common-protos/1.17.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="proto-google-iam-v1-0.13.0" MODULE="proto-google-iam-v1-0.13.0.jar" MVN="mvn:com.google.api.grpc/proto-google-iam-v1/0.13.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="protobuf-java-util-3.11.4" MODULE="protobuf-java-util-3.11.4.jar" MVN="mvn:com.google.protobuf/protobuf-java-util/3.11.4" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="annotation-1.1.0" MODULE="annotation-1.1.0.jar" MVN="mvn:androidx.annotation/annotation/1.1.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT')" />
|
||||
<IMPORT NAME="google-api-services-storage-v1-rev20201112-1.31.0" MODULE="google-api-services-storage-v1-rev20201112-1.31.0.jar" MVN="mvn:com.google.apis/google-api-services-storage/v1-rev20201112-1.31.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="gson-2.8.6" MODULE="gson-2.8.6.jar" MVN="mvn:com.google.code.gson/gson/2.8.6" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="javax.annotation-api" MODULE="javax.annotation-api-1.3.jar" MVN="mvn:javax.annotation/javax.annotation-api/1.3" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="protobuf-java-2.5.0" MODULE="protobuf-java-2.5.0.jar" MVN="mvn:com.google.protobuf/protobuf-java/2.5.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="error_prone_annotation-2.1.3" MODULE="error_prone_annotation-2.1.3.jar" MVN="mvn:com.google.errorprone/error_prone_annotations/2.1.3" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="opencensus-api-0.24.0" MODULE="opencensus-api-0.24.0.jar" MVN="mvn:io.opencensus/opencensus-api/0.24.0" REQUIRED="true" />
|
||||
<IMPORT NAME="opencensus-contrib-http-util-0.24.0" MODULE="opencensus-contrib-http-util-0.24.0.jar" MVN="mvn:io.opencensus/opencensus-contrib-http-util/0.24.0" REQUIRED="true" />
|
||||
<IMPORT NAME="grpc-context-1.22.1" MODULE="grpc-context-1.22.1.jar" MVN="mvn:io.grpc/grpc-context/1.22.1" REQUIRED="true" />
|
||||
<IMPORT NAME="proto-google-common-protos-1.17.0" MODULE="proto-google-common-protos-1.17.0.jar" MVN="mvn:com.google.api.grpc/proto-google-common-protos/1.17.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="proto-google-iam-v1-0.13.0" MODULE="proto-google-iam-v1-0.13.0.jar" MVN="mvn:com.google.api.grpc/proto-google-iam-v1/0.13.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="protobuf-java-util-3.11.4" MODULE="protobuf-java-util-3.11.4.jar" MVN="mvn:com.google.protobuf/protobuf-java-util/3.11.4" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="annotation-1.1.0" MODULE="annotation-1.1.0.jar" MVN="mvn:androidx.annotation/annotation/1.1.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS>
|
||||
|
||||
@@ -8,11 +8,13 @@ SERVICE_ACCOUNT_CREDENTIALS_FILE.NAME=Service account credentials file
|
||||
AUTH_MODE.NAME=Authentication mode
|
||||
AUTH_MODE.ITEM.SERVICEACCOUNT=Service account
|
||||
AUTH_MODE.ITEM.OAUTH=OAuth 2.0
|
||||
AUTH_MODE.ITEM.TOKEN=OAuth Access Token
|
||||
|
||||
CLIENT_ID.NAME=Client Id
|
||||
CLIENT_SECRET.NAME=Client Secret
|
||||
PROJECT_ID.NAME=Project Id
|
||||
PROJECT_ID.NAME=Project ID
|
||||
AUTHORIZATION_CODE.NAME=Authorization Code
|
||||
ACCESS_TOKEN.NAME=OAuth Access Token
|
||||
|
||||
SCHEMA.NAME=Schema
|
||||
|
||||
@@ -50,3 +52,6 @@ AUTH_TYPE.NAME=Credential type
|
||||
AUTH_TYPE.ITEM.USER_ACCOUNT_HMAC=HMAC key (deprecated)
|
||||
AUTH_TYPE.ITEM.GS_SERVICE_ACCOUNT=Service account
|
||||
GS_SERVICE_ACCOUNT_KEY.NAME=Service account key
|
||||
|
||||
GS_ACCESS_TOKEN.NAME=OAuth Access Token
|
||||
AUTH_TYPE.ITEM.TOKEN=OAuth Access Token
|
||||
@@ -2,7 +2,7 @@ LONG_NAME=Connects and loads data efficiently from a file into Google BigQuery
|
||||
HELP=org.talend.help.tBigQueryBulkExec
|
||||
CLIENT_ID.NAME=Client Id
|
||||
CLIENT_SECRET.NAME=Client Secret
|
||||
PROJECT_ID.NAME=Project Id
|
||||
PROJECT_ID.NAME=Project ID
|
||||
AUTHORIZATION_CODE.NAME=Authorization Code
|
||||
SCHEMA.NAME=Schema
|
||||
DATASET.NAME=Dataset
|
||||
|
||||
@@ -6,24 +6,31 @@
|
||||
|
||||
class BigQueryUtil_<%=cid%> {
|
||||
String projectId;
|
||||
|
||||
|
||||
<%=basePackage%>com.google.api.services.bigquery.Bigquery bigqueryclient = null;
|
||||
<%=basePackage%>com.google.api.client.auth.oauth2.Credential credentials = null;
|
||||
String tokenFile;
|
||||
boolean useLargeResult = false;
|
||||
String tempDataset;
|
||||
String tempTable;
|
||||
final boolean isAutoGeneratedTemporaryDataset;
|
||||
|
||||
|
||||
public BigQueryUtil_<%=cid%>(String projectId, <%=basePackage%>com.google.api.services.bigquery.Bigquery bigqueryclient, String tokenFile) {
|
||||
this (projectId, bigqueryclient, tokenFile, null);
|
||||
this (projectId, bigqueryclient, tokenFile, null, null);
|
||||
}
|
||||
|
||||
public BigQueryUtil_<%=cid%>(String projectId, <%=basePackage%>com.google.api.services.bigquery.Bigquery bigqueryclient, String tokenFile, String tempDataset) {
|
||||
public BigQueryUtil_<%=cid%>(String projectId, <%=basePackage%>com.google.api.services.bigquery.Bigquery bigqueryclient, String tokenFile, String tempDataset) {
|
||||
this (projectId, bigqueryclient, tokenFile, tempDataset, null);
|
||||
}
|
||||
|
||||
public BigQueryUtil_<%=cid%>(String projectId, <%=basePackage%>com.google.api.services.bigquery.Bigquery bigqueryclient, String tokenFile, String tempDataset,
|
||||
<%=basePackage%>com.google.api.client.auth.oauth2.Credential credentials) {
|
||||
this.projectId = projectId;
|
||||
this.bigqueryclient = bigqueryclient;
|
||||
this.tokenFile = tokenFile;
|
||||
this.tempDataset = tempDataset;
|
||||
this.isAutoGeneratedTemporaryDataset = tempDataset == null;
|
||||
this.credentials = credentials;
|
||||
}
|
||||
|
||||
private String genTempName(String prefix){
|
||||
@@ -32,13 +39,21 @@ class BigQueryUtil_<%=cid%> {
|
||||
|
||||
public void cleanup() throws Exception{
|
||||
if(useLargeResult){
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
if (credentials !=null)
|
||||
credentials.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
bigqueryclient.tables().delete(projectId, tempDataset, tempTable).execute();
|
||||
if (isAutoGeneratedTemporaryDataset) {
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
if (credentials !=null)
|
||||
credentials.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
bigqueryclient.datasets().delete(projectId, tempDataset).execute();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private String getLocation(<%=basePackage%>com.google.api.services.bigquery.model.JobConfigurationQuery queryConfig) throws Exception {
|
||||
String location = null;
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.JobConfiguration config = new <%=basePackage%>com.google.api.services.bigquery.model.JobConfiguration();
|
||||
@@ -46,6 +61,10 @@ class BigQueryUtil_<%=cid%> {
|
||||
config.setDryRun(true);
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.Job job = new <%=basePackage%>com.google.api.services.bigquery.model.Job();
|
||||
job.setConfiguration(config);
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
if (credentials !=null)
|
||||
credentials.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
location = bigqueryclient.jobs().insert(projectId, job).execute().getJobReference().getLocation();
|
||||
location = location == null ? "US" : location;
|
||||
return location;
|
||||
@@ -59,13 +78,17 @@ class BigQueryUtil_<%=cid%> {
|
||||
String description = "Dataset for BigQuery query job temporary table";
|
||||
dataset.setFriendlyName(description);
|
||||
dataset.setDescription(description);
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
if (credentials !=null)
|
||||
credentials.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
bigqueryclient.datasets().insert(projectId, dataset).execute();
|
||||
}
|
||||
|
||||
|
||||
public <%=basePackage%>com.google.api.services.bigquery.model.Job executeQuery(String query, boolean useLargeResult) throws Exception{
|
||||
return executeQuery(query, useLargeResult, true);
|
||||
}
|
||||
|
||||
|
||||
public <%=basePackage%>com.google.api.services.bigquery.model.Job executeQuery(String query, boolean useLargeResult, boolean useLegacySql) throws Exception{
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.JobConfigurationQuery queryConfig = new <%=basePackage%>com.google.api.services.bigquery.model.JobConfigurationQuery();
|
||||
queryConfig.setQuery(query);
|
||||
@@ -84,16 +107,20 @@ class BigQueryUtil_<%=cid%> {
|
||||
.setDatasetId(tempDataset)
|
||||
.setTableId(tempTable));
|
||||
}
|
||||
|
||||
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.JobConfiguration config = new <%=basePackage%>com.google.api.services.bigquery.model.JobConfiguration();
|
||||
config.setQuery(queryConfig);
|
||||
|
||||
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.Job job = new <%=basePackage%>com.google.api.services.bigquery.model.Job();
|
||||
job.setConfiguration(config);
|
||||
|
||||
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.Job insert = null;
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.JobReference jobId = null;
|
||||
try {
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
if (credentials !=null)
|
||||
credentials.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
insert = bigqueryclient.jobs().insert(projectId, job).execute();
|
||||
jobId = insert.getJobReference();
|
||||
} catch (<%=basePackage%>com.google.api.client.googleapis.json.GoogleJsonResponseException e) {
|
||||
@@ -142,7 +169,10 @@ class BigQueryUtil_<%=cid%> {
|
||||
%>
|
||||
// wait for query execution
|
||||
while (true) {
|
||||
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
if (credentials !=null)
|
||||
credentials.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
<%
|
||||
if("".equals(basePackage)) {
|
||||
%>
|
||||
@@ -154,7 +184,7 @@ class BigQueryUtil_<%=cid%> {
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.JobStatus status = pollJob.getStatus();
|
||||
if (status.getState().equals("DONE")) {
|
||||
<%=basePackage%>com.google.api.services.bigquery.model.ErrorProto errorProto = status.getErrorResult();
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
String basePackage = "";
|
||||
boolean isCustomTemporaryName = ElementParameterParser.getBooleanValue(node,"__USE_CUSTOM_TEMPORARY_DATASET__");
|
||||
String tempDataset = ElementParameterParser.getValue(node,"__TEMPORARY_DATASET_NAME__");
|
||||
|
||||
|
||||
String encoding = ElementParameterParser.getValue(node,"__ENCODING__");
|
||||
String advancedSeparatorStr = ElementParameterParser.getValue(node, "__ADVANCED_SEPARATOR__");
|
||||
boolean advancedSeparator = (advancedSeparatorStr!=null&&!("").equals(advancedSeparatorStr))?("true").equals(advancedSeparatorStr):false;
|
||||
@@ -44,7 +44,7 @@
|
||||
|
||||
String tokenFile = ElementParameterParser.getValue(node,"__TOKEN_NAME__");
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
|
||||
|
||||
//Dynamic start
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
int sizeListColumns = 0;
|
||||
@@ -59,11 +59,20 @@
|
||||
}
|
||||
}
|
||||
int dynamic_index=-1;
|
||||
|
||||
|
||||
//Dynamic end
|
||||
|
||||
|
||||
if (authMode.equals("OAUTH")) {
|
||||
|
||||
|
||||
if (authMode.equals("OAUTH") || authMode.equals("TOKEN") ) {
|
||||
%>
|
||||
final String PROJECT_ID_<%=cid %> = <%=projectId %>;
|
||||
com.google.api.services.bigquery.Bigquery bigqueryclient_<%=cid%> = null;
|
||||
com.google.api.client.auth.oauth2.Credential credential_<%=cid%> = null;
|
||||
long nb_line_<%=cid%> = 0;
|
||||
final com.google.api.client.http.HttpTransport TRANSPORT_<%=cid %> = new com.google.api.client.http.javanet.NetHttpTransport();
|
||||
final com.google.api.client.json.JsonFactory JSON_FACTORY_<%=cid %> = new com.google.api.client.json.jackson2.JacksonFactory();
|
||||
<%
|
||||
if (authMode.equals("OAUTH")) {
|
||||
%>
|
||||
final String CLIENT_ID_<%=cid %> = <%=clientId %>;
|
||||
<%
|
||||
@@ -73,21 +82,17 @@
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
final String CLIENT_SECRET_<%=cid%> = "{\"web\": {\"client_id\": \""+<%=clientId%>+"\",\"client_secret\": \"" +decryptedPassword_<%=cid%>+ "\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://accounts.google.com/o/oauth2/token\"}}";
|
||||
final String PROJECT_ID_<%=cid %> = <%=projectId %>;
|
||||
|
||||
// Static variables for API scope, callback URI, and HTTP/JSON functions
|
||||
final List<String> SCOPES_<%=cid%> = java.util.Arrays.asList("https://www.googleapis.com/auth/bigquery");
|
||||
final String REDIRECT_URI_<%=cid%> = "urn:ietf:wg:oauth:2.0:oob";
|
||||
final com.google.api.client.http.HttpTransport TRANSPORT_<%=cid %> = new com.google.api.client.http.javanet.NetHttpTransport();
|
||||
final com.google.api.client.json.JsonFactory JSON_FACTORY_<%=cid %> = new com.google.api.client.json.jackson2.JacksonFactory();
|
||||
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets clientSecrets_<%=cid%> = com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets.load(
|
||||
new com.google.api.client.json.jackson2.JacksonFactory(), new java.io.InputStreamReader(new java.io.ByteArrayInputStream(
|
||||
CLIENT_SECRET_<%=cid%>.getBytes())));
|
||||
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow flow_<%=cid%> = null;
|
||||
com.google.api.services.bigquery.Bigquery bigqueryclient_<%=cid%> = null;
|
||||
long nb_line_<%=cid%> = 0;
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
@@ -129,7 +134,7 @@
|
||||
// access token.
|
||||
if (storedRefreshToken_<%=cid%> != null) {
|
||||
// Request a new Access token using the refresh token.
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleCredential credential_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder().setTransport(TRANSPORT_<%=cid%>)
|
||||
credential_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder().setTransport(TRANSPORT_<%=cid%>)
|
||||
.setJsonFactory(JSON_FACTORY_<%=cid%>).setClientSecrets(clientSecrets_<%=cid%>)
|
||||
.build().setFromTokenResponse(new com.google.api.client.auth.oauth2.TokenResponse().setRefreshToken(storedRefreshToken_<%=cid%>));
|
||||
|
||||
@@ -141,7 +146,6 @@
|
||||
<%
|
||||
}
|
||||
%>
|
||||
bigqueryclient_<%=cid%> = new com.google.api.services.bigquery.Bigquery.Builder(new com.google.api.client.http.javanet.NetHttpTransport(),new com.google.api.client.json.jackson2.JacksonFactory(),credential_<%=cid%>).setApplicationName("Talend").build();
|
||||
} else {
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
@@ -183,7 +187,7 @@
|
||||
.build();
|
||||
}
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse response_<%=cid%> = flow_<%=cid%>.newTokenRequest(authorizationCode_<%=cid%>).setRedirectUri(REDIRECT_URI_<%=cid%>).execute();
|
||||
com.google.api.client.auth.oauth2.Credential credential_<%=cid%> = flow_<%=cid%>.createAndStoreCredential(response_<%=cid%>, null);
|
||||
credential_<%=cid%> = flow_<%=cid%>.createAndStoreCredential(response_<%=cid%>, null);
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
@@ -195,17 +199,25 @@
|
||||
// Store the refresh token for future use.
|
||||
java.util.Properties storeProperties_<%=cid%> = new java.util.Properties();
|
||||
storeProperties_<%=cid%>.setProperty("refreshtoken", credential_<%=cid%>.getRefreshToken());
|
||||
java.io.FileOutputStream outputStream_<%=cid%> = new java.io.FileOutputStream(tokenFile_<%=cid %>);
|
||||
storeProperties_<%=cid%>.store(outputStream_<%=cid%>,null);
|
||||
if (outputStream_<%=cid%> != null) {
|
||||
outputStream_<%=cid%>.close();
|
||||
}
|
||||
|
||||
bigqueryclient_<%=cid%> = new com.google.api.services.bigquery.Bigquery.Builder(new com.google.api.client.http.javanet.NetHttpTransport(),new com.google.api.client.json.jackson2.JacksonFactory(),credential_<%=cid%>).build();
|
||||
try (java.io.FileOutputStream outputStream_<%=cid%> = new java.io.FileOutputStream(tokenFile_<%=cid %>)) {
|
||||
storeProperties_<%=cid%>.store(outputStream_<%=cid%>,null);
|
||||
}
|
||||
}
|
||||
}
|
||||
<%} else {
|
||||
|
||||
|
||||
if (ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) {%>
|
||||
final String decryptedAccessToken_<%=cid%> = routines.system.PasswordEncryptUtil.decryptPassword(<%=ElementParameterParser.getEncryptedValue(node, "__ACCESS_TOKEN__")%>);
|
||||
credential_<%=cid%> =
|
||||
new com.google.api.client.auth.oauth2.Credential(com.google.api.client.auth.oauth2.BearerToken.authorizationHeaderAccessMethod()).setFromTokenResponse(
|
||||
(new com.google.api.client.auth.oauth2.TokenResponse()).setAccessToken(decryptedAccessToken_<%=cid%>));
|
||||
<%} else {%>
|
||||
credential_<%=cid%> =
|
||||
new com.google.api.client.auth.oauth2.Credential(com.google.api.client.auth.oauth2.BearerToken.authorizationHeaderAccessMethod()).setFromTokenResponse(
|
||||
(new com.google.api.client.auth.oauth2.TokenResponse()).setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>));
|
||||
<%}
|
||||
} %>
|
||||
bigqueryclient_<%=cid%> = new com.google.api.services.bigquery.Bigquery.Builder(TRANSPORT_<%=cid %>, JSON_FACTORY_<%=cid %>,credential_<%=cid%>).setApplicationName("Talend").build();
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/tBigQueryInput/BigQueryInputQueryHelper.javajet"%>
|
||||
|
||||
@@ -218,11 +230,12 @@
|
||||
log.debug("<%=cid%> - Running Query: "+querySql_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
if (authMode.equals("OAUTH")) { %>
|
||||
BigQueryUtil_<%=cid%> bigQueryUtil_<%=cid%> = new BigQueryUtil_<%=cid%>(PROJECT_ID_<%=cid%>, bigqueryclient_<%=cid%>, tokenFile_<%=cid%><% if (isCustomTemporaryName) {%>, <%=tempDataset%> <%} %>);
|
||||
<%} else {%>
|
||||
BigQueryUtil_<%=cid%> bigQueryUtil_<%=cid%> = new BigQueryUtil_<%=cid%>(PROJECT_ID_<%=cid%>, bigqueryclient_<%=cid%>, null, <% if (isCustomTemporaryName) {%><%=tempDataset%> <%} else {%>null<% } %>, credential_<%=cid%>);
|
||||
<%}
|
||||
|
||||
BigQueryUtil_<%=cid%> bigQueryUtil_<%=cid%> = new BigQueryUtil_<%=cid%>(PROJECT_ID_<%=cid%>, bigqueryclient_<%=cid%>, tokenFile_<%=cid%><% if (isCustomTemporaryName) {%>, <%=tempDataset%> <%} %>);
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Try <%="LARGE".equals(resultSizeType) ? "with" : "without"%> the allowLargeResults flag");
|
||||
@@ -242,15 +255,17 @@
|
||||
//Dynamic start
|
||||
|
||||
<%
|
||||
|
||||
if(isDynamic){
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/DB/Input/BigQuery/BigQueryHelper_Oauth.javajet"%>
|
||||
BigQueryHelper_<%=cid%> helper_<%=cid%> = new BigQueryHelper_<%=cid%>();
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
credential_<%=cid%>.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
com.google.api.services.bigquery.model.TableSchema schema_<%=cid%> = bigqueryclient_<%=cid%>
|
||||
.jobs().getQueryResults(PROJECT_ID_<%=cid%>, insert_<%=cid%>.getJobReference().getJobId()).execute().getSchema();
|
||||
routines.system.Dynamic dcg_<%=cid%> = new routines.system.Dynamic();
|
||||
|
||||
|
||||
<%
|
||||
|
||||
for(int i=0; i< columnList.size(); i++) {
|
||||
@@ -272,7 +287,7 @@
|
||||
dcg_<%=cid%>.metadatas.add(dynamicMetadata_<%=cid%>);
|
||||
}
|
||||
<%
|
||||
|
||||
|
||||
}
|
||||
%>
|
||||
//Dynamic end
|
||||
@@ -375,14 +390,14 @@
|
||||
//Dynamic start
|
||||
|
||||
<%
|
||||
|
||||
|
||||
if(isDynamic){
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/DB/Input/BigQuery/BigQueryHelper_ServiceAccount.javajet"%>
|
||||
BigQueryHelper_<%=cid%> helper_<%=cid%> = new BigQueryHelper_<%=cid%>();
|
||||
com.google.cloud.bigquery.Schema schema_<%=cid%> = result_<%=cid%>.getSchema();
|
||||
routines.system.Dynamic dcg_<%=cid%> = new routines.system.Dynamic();
|
||||
|
||||
|
||||
<%
|
||||
|
||||
for(int i=0; i< columnList.size(); i++) {
|
||||
@@ -411,21 +426,24 @@
|
||||
dcg_<%=cid%>.metadatas.add(dynamicMetadata_<%=cid%>);
|
||||
}
|
||||
<%
|
||||
|
||||
|
||||
}
|
||||
%>
|
||||
//Dynamic end
|
||||
long nb_line_<%=cid%> = 0;
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\", \"OAUTH\" or \"TOKEN\", but it is " + authMode);
|
||||
}
|
||||
|
||||
|
||||
if (authMode.equals("OAUTH")) {
|
||||
if (authMode.equals("OAUTH") || authMode.equals("TOKEN")) {
|
||||
%>
|
||||
while (true) {
|
||||
// Fetch Results
|
||||
<% if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
credential_<%=cid%>.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
com.google.api.services.bigquery.model.TableDataList dataList_<%=cid %> = bigqueryclient_<%=cid%>.tabledata()
|
||||
.list(PROJECT_ID_<%=cid %>,
|
||||
insert_<%=cid %>.getConfiguration().getQuery()
|
||||
@@ -452,7 +470,7 @@
|
||||
nb_line_<%=cid%> ++;
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\", \"OAUTH\" or \"TOKEN\", but it is " + authMode);
|
||||
}
|
||||
%>
|
||||
int fieldsCount_<%=cid %> = field_<%=cid %>.size();
|
||||
@@ -466,8 +484,8 @@
|
||||
IConnection conn =conns.get(0);
|
||||
String connName = conn.getName();
|
||||
if (conn.getLineStyle().hasConnectionCategory(IConnectionCategory.DATA)) {
|
||||
|
||||
|
||||
|
||||
|
||||
if ((metadatas!=null) && (metadatas.size() > 0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
if (metadata != null) {
|
||||
@@ -500,7 +518,7 @@
|
||||
<%=connName%>.<%=column.getLabel()%> = <%=defVal%>;
|
||||
} else {
|
||||
<%
|
||||
if (authMode.equals("OAUTH")) {
|
||||
if (authMode.equals("OAUTH") || authMode.equals("TOKEN")) {
|
||||
%>
|
||||
value_<%=cid%> = field_<%=cid %>.get(column_index_<%=cid%>).getV();
|
||||
<%
|
||||
@@ -509,7 +527,7 @@
|
||||
value_<%=cid%> = field_<%=cid %>.get(column_index_<%=cid%>).getValue();
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\", \"OAUTH\" or \"TOKEN\", but it is " + authMode);
|
||||
}
|
||||
%>
|
||||
if(com.google.api.client.util.Data.isNull(value_<%=cid%>)) value_<%=cid%> = null;
|
||||
|
||||
@@ -15,7 +15,7 @@ imports="
|
||||
String resultSizeType = ElementParameterParser.getValue(node,"__RESULT_SIZE__");
|
||||
boolean isCustomTemporaryName = ElementParameterParser.getBooleanValue(node,"__USE_CUSTOM_TEMPORARY_DATASET__");
|
||||
|
||||
if (authMode.equals("OAUTH")) {
|
||||
if (authMode.equals("OAUTH") || authMode.equals("TOKEN")) {
|
||||
%>
|
||||
}
|
||||
pageToken_<%=cid%> = dataList_<%=cid %>.getPageToken();
|
||||
@@ -41,7 +41,7 @@ imports="
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\" or \"TOKEN\", but it is " + authMode);
|
||||
}
|
||||
log4jFileUtil.retrievedDataNumberInfo(node);
|
||||
%>
|
||||
|
||||
@@ -43,21 +43,43 @@
|
||||
REQUIRED="true"
|
||||
NUM_ROW="1"
|
||||
/>
|
||||
|
||||
|
||||
<PARAMETER NAME="AUTH_MODE" FIELD="CLOSED_LIST" NUM_ROW="3" REQUIRED="true" GROUP="AUTHENTICATION" REPOSITORY_VALUE="AUTH_MODE">
|
||||
<ITEMS DEFAULT="SERVICEACCOUNT">
|
||||
<ITEM NAME="SERVICEACCOUNT" VALUE="SERVICEACCOUNT" />
|
||||
<ITEM NAME="OAUTH" VALUE="OAUTH" />
|
||||
<ITEM NAME="TOKEN" VALUE="TOKEN" />
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="SERVICE_ACCOUNT_CREDENTIALS_FILE"
|
||||
|
||||
<PARAMETER
|
||||
NAME="ACCESS_TOKEN"
|
||||
FIELD="PASSWORD"
|
||||
NUM_ROW="10"
|
||||
REQUIRED="true"
|
||||
SHOW_IF="AUTH_MODE == 'TOKEN'"
|
||||
GROUP="AUTHENTICATION"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="ACCESS_TOKEN_WARNING"
|
||||
FIELD="LABEL"
|
||||
NUM_ROW="11"
|
||||
REQUIRED="false"
|
||||
GROUP="AUTHENTICATION"
|
||||
SHOW_IF="AUTH_MODE == 'TOKEN'"
|
||||
>
|
||||
<DEFAULT>*Note: If the Access Token Expire we won't be able to refresh it!"</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER
|
||||
NAME="SERVICE_ACCOUNT_CREDENTIALS_FILE"
|
||||
REPOSITORY_VALUE="SERVICE_ACCOUNT_CREDENTIALS_FILE"
|
||||
GROUP="AUTHENTICATION"
|
||||
FIELD="FILE"
|
||||
NUM_ROW="10"
|
||||
REQUIRED="true"
|
||||
FIELD="FILE"
|
||||
NUM_ROW="10"
|
||||
REQUIRED="true"
|
||||
SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
@@ -104,7 +126,7 @@
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
|
||||
<PARAMETER
|
||||
NAME="USE_LEGACY_SQL"
|
||||
FIELD="CHECK"
|
||||
@@ -113,7 +135,7 @@
|
||||
>
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
|
||||
<PARAMETER
|
||||
NAME="QUERY"
|
||||
FIELD="MEMO_SQL"
|
||||
@@ -122,13 +144,13 @@
|
||||
>
|
||||
<DEFAULT>"select id, name from employee"</DEFAULT>
|
||||
</PARAMETER>
|
||||
<!--
|
||||
https://cloud.google.com/bigquery/querying-data#large-results
|
||||
<!--
|
||||
https://cloud.google.com/bigquery/querying-data#large-results
|
||||
SMALL: allowLargeResult is false
|
||||
LARGE: allowLargeResult is true and auto create/clean temp dataset/table
|
||||
AUTO: try SMALL first, if get responseTooLarge error then try LARGE
|
||||
-->
|
||||
<PARAMETER
|
||||
<PARAMETER
|
||||
NAME="RESULT_SIZE"
|
||||
FIELD="CLOSED_LIST"
|
||||
NUM_ROW="60"
|
||||
@@ -179,16 +201,19 @@
|
||||
</ADVANCED_PARAMETERS>
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="google-api-client-1.25.0.jar" MODULE="google-api-client-1.25.0.jar" MVN="mvn:com.google.api-client/google-api-client/1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-services-oauth2-v2-rev151-1.25.0.jar" MODULE="google-api-services-oauth2-v2-rev151-1.25.0.jar" MVN="mvn:com.google.apis/google-api-services-oauth2/v2-rev151-1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-services-bigquery-v2-rev454-1.25.0.jar" MODULE="google-api-services-bigquery-v2-rev454-1.25.0.jar" MVN="mvn:com.google.apis/google-api-services-bigquery/v2-rev454-1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-1.25.0.jar" MODULE="google-http-client-1.25.0.jar" MVN="mvn:com.google.http-client/google-http-client/1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-oauth-client-1.25.0.jar" MODULE="google-oauth-client-1.25.0.jar" MVN="mvn:com.google.oauth-client/google-oauth-client/1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-jackson2-1.25.0.jar" MODULE="google-http-client-jackson2-1.25.0.jar" MVN="mvn:com.google.http-client/google-http-client-jackson2/1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-client-1.31.1.jar" MODULE="google-api-client-1.31.1.jar" MVN="mvn:com.google.api-client/google-api-client/1.31.1" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-services-oauth2-v2-rev20200213-1.31.0.jar" MODULE="google-api-services-oauth2-v2-rev20200213-1.31.0.jar" MVN="mvn:com.google.apis/google-api-services-oauth2/v2-rev20200213-1.31.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-services-bigquery-v2-rev20201030-1.31.0.jar" MODULE="google-api-services-bigquery-v2-rev20201030-1.31.0.jar" MVN="mvn:com.google.apis/google-api-services-bigquery/v2-rev20201030-1.31.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-1.38.0.jar" MODULE="google-http-client-1.38.0.jar" MVN="mvn:com.google.http-client/google-http-client/1.38.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-oauth-client-1.31.0.jar" MODULE="google-oauth-client-1.31.0.jar" MVN="mvn:com.google.oauth-client/google-oauth-client/1.31.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-jackson2-1.38.0.jar" MODULE="google-http-client-jackson2-1.38.0.jar" MVN="mvn:com.google.http-client/google-http-client-jackson2/1.38.0" REQUIRED="true" />
|
||||
<IMPORT NAME="guava-20.0.jar" MODULE="guava-20.0.jar" MVN="mvn:com.google.guava/guava/20.0" REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1" REQUIRED="true" />
|
||||
<IMPORT NAME="google-cloud-bigquery-1.60.0.jar" MODULE="google-cloud-bigquery-1.60.0.jar" MVN="mvn:com.google.cloud/google-cloud-bigquery/1.60.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="jackson-core-2.11.4.jar" MODULE="jackson-core-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4" REQUIRED="true" />
|
||||
<IMPORT NAME="google-cloud-bigquery-1.126.3.jar" MODULE="google-cloud-bigquery-1.126.3.jar" MVN="mvn:com.google.cloud/google-cloud-bigquery/1.126.3" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
|
||||
<!-- TODO remove it -->
|
||||
<IMPORT NAME="google-http-client-jackson-1.25.0.jar" MODULE="google-http-client-jackson-1.25.0.jar" MVN="mvn:com.google.http-client/google-http-client-jackson/1.25.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
|
||||
<IMPORT NAME="threetenbp-1.3.3.jar" MODULE="threetenbp-1.3.3.jar" MVN="mvn:org.threeten/threetenbp/1.3.3" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="google-auth-library-credentials-0.20.0.jar" MODULE="google-auth-library-credentials-0.20.0.jar" MVN="mvn:com.google.auth/google-auth-library-credentials/0.20.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="gax-httpjson-0.44.0.jar" MODULE="gax-httpjson-0.44.0.jar" MVN="mvn:com.google.api/gax-httpjson/0.44.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
@@ -197,8 +222,11 @@
|
||||
<IMPORT NAME="google-cloud-core-1.93.4.jar" MODULE="google-cloud-core-1.93.4.jar" MVN="mvn:com.google.cloud/google-cloud-core/1.93.4" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="google-cloud-core-http-1.32.0.jar" MODULE="google-cloud-core-http-1.32.0.jar" MVN="mvn:com.google.cloud/google-cloud-core-http/1.32.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="gax-1.27.0.jar" MODULE="gax-1.27.0.jar" MVN="mvn:com.google.api/gax/1.27.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="google-http-client-appengine-1.25.0.jar" MODULE="google-http-client-appengine-1.25.0.jar" MVN="mvn:com.google.http-client/google-http-client-appengine/1.25.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="google-http-client-appengine-1.38.0.jar" MODULE="google-http-client-appengine-1.38.0.jar" MVN="mvn:com.google.http-client/google-http-client-appengine/1.38.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="api-common-1.6.0.jar" MODULE="api-common-1.6.0.jar" MVN="mvn:com.google.api/api-common/1.6.0" REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'" />
|
||||
<IMPORT NAME="opencensus-api-0.24.0" MODULE="opencensus-api-0.24.0.jar" MVN="mvn:io.opencensus/opencensus-api/0.24.0" REQUIRED="true" />
|
||||
<IMPORT NAME="opencensus-contrib-http-util-0.24.0" MODULE="opencensus-contrib-http-util-0.24.0.jar" MVN="mvn:io.opencensus/opencensus-contrib-http-util/0.24.0" REQUIRED="true" />
|
||||
<IMPORT NAME="grpc-context-1.22.1" MODULE="grpc-context-1.22.1.jar" MVN="mvn:io.grpc/grpc-context/1.22.1" REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#Created by JInto - www.guh-software.de
|
||||
#Wed Mar 19 09:39:53 CST 2008
|
||||
LONG_NAME=Connect and run a query on Google BigQuery
|
||||
LONG_NAME=Connect and run a query on Google BigQuery
|
||||
HELP=org.talend.help.tBigQueryInput
|
||||
|
||||
AUTHENTICATION.NAME=Authentication
|
||||
@@ -8,11 +8,14 @@ SERVICE_ACCOUNT_CREDENTIALS_FILE.NAME=Service account credentials file
|
||||
AUTH_MODE.NAME=Authentication mode
|
||||
AUTH_MODE.ITEM.SERVICEACCOUNT=Service account
|
||||
AUTH_MODE.ITEM.OAUTH=OAuth 2.0
|
||||
AUTH_MODE.ITEM.TOKEN=OAuth Access Token
|
||||
|
||||
CLIENT_ID.NAME=Client Id
|
||||
CLIENT_SECRET.NAME=Client Secret
|
||||
PROJECT_ID.NAME=Project Id
|
||||
PROJECT_ID.NAME=Project ID
|
||||
AUTHORIZATION_CODE.NAME=Authorization Code
|
||||
ACCESS_TOKEN.NAME=OAuth Access Token
|
||||
|
||||
QUERY.NAME=Query
|
||||
SCHEMA.NAME=Schema
|
||||
|
||||
@@ -29,4 +32,4 @@ RESULT_SIZE.ITEM.LARGE=Large(with allowLargeResults)
|
||||
RESULT_SIZE.ITEM.AUTO=Auto
|
||||
USE_CUSTOM_TEMPORARY_DATASET.NAME=Use custom temporary Dataset name
|
||||
TEMPORARY_DATASET_NAME.NAME=Temporary Dataset name
|
||||
USE_LEGACY_SQL.NAME=Use Legacy SQL
|
||||
USE_LEGACY_SQL.NAME=Use Legacy SQL
|
||||
@@ -2,7 +2,7 @@ LONG_NAME=Connect and run a query on Google BigQuery
|
||||
HELP=org.talend.help.tBigQueryInput
|
||||
CLIENT_ID.NAME=Client Id
|
||||
CLIENT_SECRET.NAME=Client Secret
|
||||
PROJECT_ID.NAME=Project Id
|
||||
PROJECT_ID.NAME=Project ID
|
||||
AUTHORIZATION_CODE.NAME=Authorization Code
|
||||
QUERY.NAME=Query
|
||||
SCHEMA.NAME=Schema
|
||||
|
||||
@@ -54,20 +54,43 @@
|
||||
<PARAMETER NAME="SCHEMA" FIELD="SCHEMA_TYPE" REQUIRED="true" NUM_ROW="13">
|
||||
<DEFAULT/>
|
||||
</PARAMETER>
|
||||
|
||||
|
||||
<PARAMETER NAME="AUTH_MODE" FIELD="CLOSED_LIST" NUM_ROW="15" REQUIRED="true" GROUP="AUTHENTICATION" REPOSITORY_VALUE="AUTH_MODE">
|
||||
<ITEMS DEFAULT="SERVICEACCOUNT">
|
||||
<ITEM NAME="SERVICEACCOUNT" VALUE="SERVICEACCOUNT" />
|
||||
<ITEM NAME="OAUTH" VALUE="OAUTH" />
|
||||
<ITEM NAME="TOKEN" VALUE="TOKEN" />
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="SERVICE_ACCOUNT_CREDENTIALS_FILE"
|
||||
<PARAMETER
|
||||
NAME="ACCESS_TOKEN"
|
||||
FIELD="PASSWORD"
|
||||
NUM_ROW="18"
|
||||
REQUIRED="true"
|
||||
SHOW_IF="AUTH_MODE == 'TOKEN'"
|
||||
GROUP="AUTHENTICATION"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="ACCESS_TOKEN_WARNING"
|
||||
FIELD="LABEL"
|
||||
NUM_ROW="20"
|
||||
REQUIRED="false"
|
||||
GROUP="AUTHENTICATION"
|
||||
SHOW_IF="AUTH_MODE == 'TOKEN'"
|
||||
>
|
||||
<DEFAULT>*Note: If the Access Token Expire we won't be able to refresh it!"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="SERVICE_ACCOUNT_CREDENTIALS_FILE"
|
||||
REPOSITORY_VALUE="SERVICE_ACCOUNT_CREDENTIALS_FILE"
|
||||
FIELD="FILE"
|
||||
NUM_ROW="18"
|
||||
REQUIRED="true"
|
||||
FIELD="FILE"
|
||||
NUM_ROW="18"
|
||||
REQUIRED="true"
|
||||
SHOW_IF="AUTH_MODE == 'SERVICEACCOUNT'"
|
||||
GROUP="AUTHENTICATION"
|
||||
>
|
||||
@@ -162,25 +185,26 @@
|
||||
<ITEMS DEFAULT="GS_SERVICE_ACCOUNT">
|
||||
<ITEM NAME="USER_ACCOUNT_HMAC" VALUE="USER_ACCOUNT_HMAC"/>
|
||||
<ITEM NAME="GS_SERVICE_ACCOUNT" VALUE="GS_SERVICE_ACCOUNT"/>
|
||||
<ITEM NAME="TOKEN" VALUE="TOKEN" />
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="GS_ACCESS_KEY"
|
||||
FIELD="TEXT"
|
||||
NUM_ROW="60"
|
||||
REQUIRED="true"
|
||||
GROUP="GS_CONF"
|
||||
<PARAMETER
|
||||
NAME="GS_ACCESS_KEY"
|
||||
FIELD="TEXT"
|
||||
NUM_ROW="60"
|
||||
REQUIRED="true"
|
||||
GROUP="GS_CONF"
|
||||
SHOW_IF="(BULK_FILE_ALREADY_EXIST=='false') AND (AUTH_TYPE == 'USER_ACCOUNT_HMAC')">
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="GS_SECRET_KEY"
|
||||
FIELD="PASSWORD"
|
||||
NUM_ROW="61"
|
||||
REQUIRED="true"
|
||||
GROUP="GS_CONF"
|
||||
<PARAMETER
|
||||
NAME="GS_SECRET_KEY"
|
||||
FIELD="PASSWORD"
|
||||
NUM_ROW="61"
|
||||
REQUIRED="true"
|
||||
GROUP="GS_CONF"
|
||||
SHOW_IF="(BULK_FILE_ALREADY_EXIST=='false') AND (AUTH_TYPE == 'USER_ACCOUNT_HMAC')">
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
@@ -195,6 +219,28 @@
|
||||
<DEFAULT>"__COMP_DEFAULT_FILE_DIR__/key.json"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="GS_ACCESS_TOKEN"
|
||||
FIELD="PASSWORD"
|
||||
NUM_ROW="60"
|
||||
REQUIRED_IF="(BULK_FILE_ALREADY_EXIST=='false') AND (AUTH_TYPE == 'TOKEN')"
|
||||
SHOW_IF="(BULK_FILE_ALREADY_EXIST=='false') AND (AUTH_TYPE == 'TOKEN')"
|
||||
GROUP="GS_CONF"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="GS_ACCESS_TOKEN_WARNING"
|
||||
FIELD="LABEL"
|
||||
NUM_ROW="61"
|
||||
REQUIRED="false"
|
||||
GROUP="GS_CONF"
|
||||
SHOW_IF="(BULK_FILE_ALREADY_EXIST=='false') AND (AUTH_TYPE == 'TOKEN')"
|
||||
>
|
||||
<DEFAULT>*Note: If the Access Token Expire we won't be able to refresh it!"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="BUCKET_NAME" FIELD="TEXT" NUM_ROW="63" REQUIRED="true" GROUP="GS_CONF" SHOW_IF="BULK_FILE_ALREADY_EXIST=='false'">
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
@@ -303,6 +349,8 @@
|
||||
|
||||
<TEMPLATE_PARAM SOURCE="self.AUTH_MODE" TARGET="tBQBE.AUTH_MODE" />
|
||||
<TEMPLATE_PARAM SOURCE="self.SERVICE_ACCOUNT_CREDENTIALS_FILE" TARGET="tBQBE.SERVICE_ACCOUNT_CREDENTIALS_FILE" />
|
||||
<TEMPLATE_PARAM SOURCE="self.ACCESS_TOKEN" TARGET="tBQBE.ACCESS_TOKEN" />
|
||||
<TEMPLATE_PARAM SOURCE="self.ACCESS_TOKEN_WARNING" TARGET="tBQBE.ACCESS_TOKEN_WARNING" />
|
||||
<TEMPLATE_PARAM SOURCE="self.CLIENT_ID" TARGET="tBQBE.CLIENT_ID" />
|
||||
<TEMPLATE_PARAM SOURCE="self.CLIENT_SECRET" TARGET="tBQBE.CLIENT_SECRET" />
|
||||
<TEMPLATE_PARAM SOURCE="self.PROJECT_ID" TARGET="tBQBE.PROJECT_ID" />
|
||||
@@ -313,6 +361,8 @@
|
||||
<TEMPLATE_PARAM SOURCE="self.ACTION_ON_DATA" TARGET="tBQBE.ACTION_ON_DATA" />
|
||||
<TEMPLATE_PARAM SOURCE="self.AUTH_TYPE" TARGET="tBQBE.AUTH_TYPE" />
|
||||
<TEMPLATE_PARAM SOURCE="self.GS_SERVICE_ACCOUNT_KEY" TARGET="tBQBE.GS_SERVICE_ACCOUNT_KEY" />
|
||||
<TEMPLATE_PARAM SOURCE="self.GS_ACCESS_TOKEN" TARGET="tBQBE.GS_ACCESS_TOKEN" />
|
||||
<TEMPLATE_PARAM SOURCE="self.GS_ACCESS_TOKEN_WARNING" TARGET="tBQBE.GS_ACCESS_TOKEN_WARNING" />
|
||||
<TEMPLATE_PARAM SOURCE="self.GS_ACCESS_KEY" TARGET="tBQBE.GS_ACCESS_KEY" />
|
||||
<TEMPLATE_PARAM SOURCE="self.GS_SECRET_KEY" TARGET="tBQBE.GS_SECRET_KEY" />
|
||||
<TEMPLATE_PARAM SOURCE="self.BUCKET_NAME" TARGET="tBQBE.BUCKET_NAME" />
|
||||
|
||||
@@ -22,11 +22,13 @@ SERVICE_ACCOUNT_CREDENTIALS_FILE.NAME=Service account credentials file
|
||||
AUTH_MODE.NAME=Authentication mode
|
||||
AUTH_MODE.ITEM.SERVICEACCOUNT=Service account
|
||||
AUTH_MODE.ITEM.OAUTH=OAuth 2.0
|
||||
AUTH_MODE.ITEM.TOKEN=OAuth Access Token
|
||||
|
||||
CLIENT_ID.NAME=Client Id
|
||||
CLIENT_SECRET.NAME=Client Secret
|
||||
PROJECT_ID.NAME=Project Id
|
||||
PROJECT_ID.NAME=Project ID
|
||||
AUTHORIZATION_CODE.NAME=Authorization Code
|
||||
ACCESS_TOKEN.NAME=OAuth Access Token
|
||||
|
||||
SCHEMA.NAME=Schema
|
||||
|
||||
@@ -62,4 +64,6 @@ DIE_ON_ERROR.NAME=Die on error
|
||||
AUTH_TYPE.NAME=Credential type
|
||||
AUTH_TYPE.ITEM.USER_ACCOUNT_HMAC=HMAC key (deprecated)
|
||||
AUTH_TYPE.ITEM.GS_SERVICE_ACCOUNT=Service account
|
||||
AUTH_TYPE.ITEM.TOKEN=OAuth Access Token
|
||||
GS_SERVICE_ACCOUNT_KEY.NAME=Service account key
|
||||
GS_ACCESS_TOKEN.NAME=OAuth Access Token
|
||||
|
||||
@@ -16,7 +16,7 @@ LONG.NAME=Connects and loads data from a flow into Google BigQuery
|
||||
HELP=org.talend.help.tBigQueryOutput
|
||||
CLIENT_ID.NAME=Client Id
|
||||
CLIENT_SECRET.NAME=Client Secret
|
||||
PROJECT_ID.NAME=Project Id
|
||||
PROJECT_ID.NAME=Project ID
|
||||
AUTHORIZATION_CODE.NAME=Authorization Code
|
||||
SCHEMA.NAME=Schema
|
||||
DATASET.NAME=Dataset
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<%@ jet
|
||||
imports="
|
||||
org.talend.core.model.process.INode
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
org.talend.core.model.process.INode
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
org.talend.designer.codegen.config.CodeGeneratorArgument
|
||||
java.util.List
|
||||
"
|
||||
@@ -23,33 +23,39 @@
|
||||
String query = ElementParameterParser.getValue(node,"__QUERY__");
|
||||
boolean useLegacySql = ElementParameterParser.getBooleanValue(node,"__USE_LEGACY_SQL__");
|
||||
String basePackage = "";
|
||||
|
||||
|
||||
query = query.replaceAll("\n"," ").replaceAll("\r", " ");
|
||||
|
||||
|
||||
String tokenFile = ElementParameterParser.getValue(node,"__TOKEN_NAME__");
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
if (authMode.equals("OAUTH")) {
|
||||
if (authMode.equals("OAUTH") || authMode.equals("TOKEN")) {
|
||||
%>
|
||||
final String PROJECT_ID_<%=cid %> = <%=projectId %>;
|
||||
final com.google.api.client.http.HttpTransport TRANSPORT_<%=cid %> = new com.google.api.client.http.javanet.NetHttpTransport();
|
||||
final com.google.api.client.json.JsonFactory JSON_FACTORY_<%=cid %> = new com.google.api.client.json.jackson2.JacksonFactory();
|
||||
com.google.api.services.bigquery.Bigquery bigqueryclient_<%=cid%> = null;
|
||||
com.google.api.client.auth.oauth2.Credential credential_<%=cid%> = null;
|
||||
long nb_line_<%=cid%> = 0;
|
||||
<%
|
||||
if (authMode.equals("OAUTH")) {
|
||||
String passwordFieldName = "__CLIENT_SECRET__";
|
||||
%>
|
||||
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/password.javajet"%>
|
||||
|
||||
|
||||
final String CLIENT_SECRET_<%=cid%> = "{\"web\": {\"client_id\": \""+<%=clientId%>+"\",\"client_secret\": \"" +decryptedPassword_<%=cid%>+ "\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://accounts.google.com/o/oauth2/token\"}}";
|
||||
final String PROJECT_ID_<%=cid %> = <%=projectId %>;
|
||||
|
||||
|
||||
|
||||
// Static variables for API scope, callback URI, and HTTP/JSON functions
|
||||
final List<String> SCOPES_<%=cid%> = java.util.Arrays.asList("https://www.googleapis.com/auth/bigquery");
|
||||
final String REDIRECT_URI_<%=cid%> = "urn:ietf:wg:oauth:2.0:oob";
|
||||
final com.google.api.client.http.HttpTransport TRANSPORT_<%=cid %> = new com.google.api.client.http.javanet.NetHttpTransport();
|
||||
final com.google.api.client.json.JsonFactory JSON_FACTORY_<%=cid %> = new com.google.api.client.json.jackson2.JacksonFactory();
|
||||
|
||||
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets clientSecrets_<%=cid%> = com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets.load(
|
||||
JSON_FACTORY_<%=cid %>, new java.io.InputStreamReader(new java.io.ByteArrayInputStream(
|
||||
CLIENT_SECRET_<%=cid%>.getBytes())));
|
||||
|
||||
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow flow_<%=cid%> = null;
|
||||
com.google.api.services.bigquery.Bigquery bigqueryclient_<%=cid%> = null;
|
||||
long nb_line_<%=cid%> = 0;
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
@@ -59,7 +65,7 @@
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
|
||||
// Attempt to load existing refresh token
|
||||
String tokenFile_<%=cid %> = <%=tokenFile%>;
|
||||
java.util.Properties properties_<%=cid%> = new java.util.Properties();
|
||||
@@ -75,16 +81,16 @@
|
||||
%>
|
||||
}
|
||||
String storedRefreshToken_<%=cid%> = (String) properties_<%=cid%>.get("refreshtoken");
|
||||
|
||||
|
||||
// Check to see if the an existing refresh token was loaded.
|
||||
// If so, create a credential and call refreshToken() to get a new
|
||||
// access token.
|
||||
if (storedRefreshToken_<%=cid%> != null) {
|
||||
// Request a new Access token using the refresh token.
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleCredential credential_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2. GoogleCredential.Builder().setTransport(TRANSPORT_<%=cid%>)
|
||||
credential_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2.GoogleCredential.Builder().setTransport(TRANSPORT_<%=cid%>)
|
||||
.setJsonFactory(JSON_FACTORY_<%=cid%>).setClientSecrets(clientSecrets_<%=cid%>)
|
||||
.build().setFromTokenResponse(new com.google.api.client.auth.oauth2.TokenResponse().setRefreshToken(storedRefreshToken_<%=cid%>));
|
||||
|
||||
|
||||
credential_<%=cid%>.refreshToken();
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
@@ -93,7 +99,6 @@
|
||||
<%
|
||||
}
|
||||
%>
|
||||
bigqueryclient_<%=cid%> = new com.google.api.services.bigquery.Bigquery.Builder(new com.google.api.client.http.javanet.NetHttpTransport(),new com.google.api.client.json.jackson2.JacksonFactory(),credential_<%=cid%>).setApplicationName("Talend").build();
|
||||
} else {
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
@@ -106,7 +111,7 @@
|
||||
if(authorizationCode_<%=cid%> == null || "".equals(authorizationCode_<%=cid%>) || "\"\"".equals(authorizationCode_<%=cid%>)) {
|
||||
String authorizeUrl_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeRequestUrl(
|
||||
clientSecrets_<%=cid%>, REDIRECT_URI_<%=cid%>, SCOPES_<%=cid%>).setState("").build();
|
||||
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
@@ -131,13 +136,12 @@
|
||||
}
|
||||
%>
|
||||
// Exchange the auth code for an access token and refesh token
|
||||
flow_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow.Builder(new com.google.api.client.http.javanet.NetHttpTransport(),
|
||||
new com.google.api.client.json.jackson2.JacksonFactory(), clientSecrets_<%=cid%>, SCOPES_<%=cid%>)
|
||||
flow_<%=cid%> = new com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow.Builder(TRANSPORT_<%=cid %>, JSON_FACTORY_<%=cid%>, clientSecrets_<%=cid%>, SCOPES_<%=cid%>)
|
||||
.setAccessType("offline").setApprovalPrompt("force")
|
||||
.build();
|
||||
com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse response_<%=cid%> = flow_<%=cid%>.newTokenRequest(authorizationCode_<%=cid%>).setRedirectUri(REDIRECT_URI_<%=cid%>).execute();
|
||||
com.google.api.client.auth.oauth2.Credential credential_<%=cid%> = flow_<%=cid%>.createAndStoreCredential(response_<%=cid%>, null);
|
||||
|
||||
credential_<%=cid%> = flow_<%=cid%>.createAndStoreCredential(response_<%=cid%>, null);
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
@@ -151,17 +155,28 @@
|
||||
try(java.io.FileOutputStream outputStream_<%=cid%> = new java.io.FileOutputStream(tokenFile_<%=cid %>)) {
|
||||
storeProperties_<%=cid%>.store(outputStream_<%=cid%>,null);
|
||||
}
|
||||
|
||||
bigqueryclient_<%=cid%> = new com.google.api.services.bigquery.Bigquery.Builder(new com.google.api.client.http.javanet.NetHttpTransport(),new com.google.api.client.json.jackson2.JacksonFactory(),credential_<%=cid%>).build();
|
||||
}
|
||||
}
|
||||
<% } else {
|
||||
if (ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) {%>
|
||||
final String decryptedAccessToken_<%=cid%> = routines.system.PasswordEncryptUtil.decryptPassword(<%=ElementParameterParser.getEncryptedValue(node, "__ACCESS_TOKEN__")%>);
|
||||
credential_<%=cid%> =
|
||||
new com.google.api.client.auth.oauth2.Credential(com.google.api.client.auth.oauth2.BearerToken.authorizationHeaderAccessMethod()).setFromTokenResponse(
|
||||
(new com.google.api.client.auth.oauth2.TokenResponse()).setAccessToken(decryptedAccessToken_<%=cid%>));
|
||||
<%} else {%>
|
||||
credential_<%=cid%> =
|
||||
new com.google.api.client.auth.oauth2.Credential(com.google.api.client.auth.oauth2.BearerToken.authorizationHeaderAccessMethod()).setFromTokenResponse(
|
||||
(new com.google.api.client.auth.oauth2.TokenResponse()).setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>));
|
||||
<%}
|
||||
} %>
|
||||
bigqueryclient_<%=cid%> = new com.google.api.services.bigquery.Bigquery.Builder(TRANSPORT_<%=cid %>, JSON_FACTORY_<%=cid%>, credential_<%=cid%>).setApplicationName("Talend").build();
|
||||
<%
|
||||
String resultSizeType = ElementParameterParser.getValue(node,"__RESULT_SIZE__");
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/tBigQueryInput/BigQueryInputQueryHelper.javajet"%>
|
||||
|
||||
BigQueryUtil_<%=cid%> bigQueryUtil_<%=cid%> = new BigQueryUtil_<%=cid%>(PROJECT_ID_<%=cid%>, bigqueryclient_<%=cid%>, tokenFile_<%=cid%>);
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/tBigQueryInput/BigQueryInputQueryHelper.javajet"%>
|
||||
|
||||
BigQueryUtil_<%=cid%> bigQueryUtil_<%=cid%> = new BigQueryUtil_<%=cid%>(PROJECT_ID_<%=cid%>, bigqueryclient_<%=cid%>, <% if (authMode.equals("OAUTH")) { %> tokenFile_<%=cid%> <% } else { %> null <%}%>);
|
||||
<%
|
||||
} else if (authMode.equals("SERVICEACCOUNT")) {
|
||||
%>
|
||||
@@ -176,7 +191,7 @@
|
||||
.setProjectId(<%=projectId%>)
|
||||
.build()
|
||||
.getService();
|
||||
|
||||
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
|
||||
@@ -44,9 +44,31 @@
|
||||
<ITEMS DEFAULT="SERVICEACCOUNT">
|
||||
<ITEM NAME="SERVICEACCOUNT" VALUE="SERVICEACCOUNT"/>
|
||||
<ITEM NAME="OAUTH" VALUE="OAUTH"/>
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
<ITEM NAME="TOKEN" VALUE="TOKEN" />
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="ACCESS_TOKEN"
|
||||
FIELD="PASSWORD"
|
||||
NUM_ROW="10"
|
||||
REQUIRED="true"
|
||||
SHOW_IF="AUTH_MODE == 'TOKEN'"
|
||||
GROUP="AUTHENTICATION"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="ACCESS_TOKEN_WARNING"
|
||||
FIELD="LABEL"
|
||||
NUM_ROW="11"
|
||||
REQUIRED="false"
|
||||
GROUP="AUTHENTICATION"
|
||||
SHOW_IF="AUTH_MODE == 'TOKEN'"
|
||||
>
|
||||
<DEFAULT>*Note: If the Access Token Expire we won't be able to refresh it!"</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER
|
||||
NAME="SERVICE_ACCOUNT_CREDENTIALS_FILE"
|
||||
REPOSITORY_VALUE="SERVICE_ACCOUNT_CREDENTIALS_FILE"
|
||||
@@ -166,21 +188,24 @@
|
||||
</ADVANCED_PARAMETERS>
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="google-api-client-1.25.0.jar" MODULE="google-api-client-1.25.0.jar" MVN="mvn:com.google.api-client/google-api-client/1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-services-oauth2-v2-rev151-1.25.0.jar" MODULE="google-api-services-oauth2-v2-rev151-1.25.0.jar" MVN="mvn:com.google.apis/google-api-services-oauth2/v2-rev151-1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-services-bigquery-v2-rev454-1.25.0.jar" MODULE="google-api-services-bigquery-v2-rev454-1.25.0.jar" MVN="mvn:com.google.apis/google-api-services-bigquery/v2-rev454-1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-1.25.0.jar" MODULE="google-http-client-1.25.0.jar" MVN="mvn:com.google.http-client/google-http-client/1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-oauth-client-1.25.0.jar" MODULE="google-oauth-client-1.25.0.jar" MVN="mvn:com.google.oauth-client/google-oauth-client/1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-jackson2-1.25.0.jar" MODULE="google-http-client-jackson2-1.25.0.jar" MVN="mvn:com.google.http-client/google-http-client-jackson2/1.25.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-client-1.31.1.jar" MODULE="google-api-client-1.31.1.jar" MVN="mvn:com.google.api-client/google-api-client/1.31.1" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-services-oauth2-v2-rev20200213-1.31.0.jar" MODULE="google-api-services-oauth2-v2-rev20200213-1.31.0.jar" MVN="mvn:com.google.apis/google-api-services-oauth2/v2-rev20200213-1.31.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-api-services-bigquery-v2-rev20201030-1.31.0.jar" MODULE="google-api-services-bigquery-v2-rev20201030-1.31.0.jar" MVN="mvn:com.google.apis/google-api-services-bigquery/v2-rev20201030-1.31.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-1.38.0.jar" MODULE="google-http-client-1.38.0.jar" MVN="mvn:com.google.http-client/google-http-client/1.38.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-oauth-client-1.31.0.jar" MODULE="google-oauth-client-1.31.0.jar" MVN="mvn:com.google.oauth-client/google-oauth-client/1.31.0" REQUIRED="true" />
|
||||
<IMPORT NAME="google-http-client-jackson2-1.38.0.jar" MODULE="google-http-client-jackson2-1.38.0.jar" MVN="mvn:com.google.http-client/google-http-client-jackson2/1.38.0" REQUIRED="true" />
|
||||
<IMPORT NAME="guava-20.0.jar" MODULE="guava-20.0.jar" MVN="mvn:com.google.guava/guava/20.0" REQUIRED="true"/>
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1" REQUIRED="true"/>
|
||||
<IMPORT NAME="google-cloud-bigquery-1.60.0.jar" MODULE="google-cloud-bigquery-1.60.0.jar"
|
||||
MVN="mvn:com.google.cloud/google-cloud-bigquery/1.60.0"
|
||||
<IMPORT NAME="jackson-core-2.11.4.jar" MODULE="jackson-core-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4" REQUIRED="true"/>
|
||||
<IMPORT NAME="google-cloud-bigquery-1.126.3.jar" MODULE="google-cloud-bigquery-1.126.3.jar"
|
||||
MVN="mvn:com.google.cloud/google-cloud-bigquery/1.126.3"
|
||||
REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'"/>
|
||||
|
||||
<!-- TODO remove it -->
|
||||
<IMPORT NAME="google-http-client-jackson-1.25.0.jar" MODULE="google-http-client-jackson-1.25.0.jar"
|
||||
MVN="mvn:com.google.http-client/google-http-client-jackson/1.25.0"
|
||||
REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'"/>
|
||||
|
||||
<IMPORT NAME="threetenbp-1.3.3.jar" MODULE="threetenbp-1.3.3.jar" MVN="mvn:org.threeten/threetenbp/1.3.3"
|
||||
REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'"/>
|
||||
<IMPORT NAME="google-auth-library-credentials-0.20.0.jar" MODULE="google-auth-library-credentials-0.20.0.jar"
|
||||
@@ -200,11 +225,14 @@
|
||||
REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'"/>
|
||||
<IMPORT NAME="gax-1.27.0.jar" MODULE="gax-1.27.0.jar" MVN="mvn:com.google.api/gax/1.27.0"
|
||||
REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'"/>
|
||||
<IMPORT NAME="google-http-client-appengine-1.25.0.jar" MODULE="google-http-client-appengine-1.25.0.jar"
|
||||
MVN="mvn:com.google.http-client/google-http-client-appengine/1.25.0"
|
||||
<IMPORT NAME="google-http-client-appengine-1.38.0.jar" MODULE="google-http-client-appengine-1.38.0.jar"
|
||||
MVN="mvn:com.google.http-client/google-http-client-appengine/1.38.0"
|
||||
REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'"/>
|
||||
<IMPORT NAME="api-common-1.6.0.jar" MODULE="api-common-1.6.0.jar" MVN="mvn:com.google.api/api-common/1.6.0"
|
||||
REQUIRED_IF="AUTH_MODE == 'SERVICEACCOUNT'"/>
|
||||
<IMPORT NAME="opencensus-api-0.24.0" MODULE="opencensus-api-0.24.0.jar" MVN="mvn:io.opencensus/opencensus-api/0.24.0" REQUIRED="true" />
|
||||
<IMPORT NAME="opencensus-contrib-http-util-0.24.0" MODULE="opencensus-contrib-http-util-0.24.0.jar" MVN="mvn:io.opencensus/opencensus-contrib-http-util/0.24.0" REQUIRED="true" />
|
||||
<IMPORT NAME="grpc-context-1.22.1" MODULE="grpc-context-1.22.1.jar" MVN="mvn:io.grpc/grpc-context/1.22.1" REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
<RETURNS>
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
<%@ jet
|
||||
imports="
|
||||
org.talend.core.model.process.INode
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
org.talend.core.model.process.INode
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
org.talend.designer.codegen.config.CodeGeneratorArgument
|
||||
org.talend.core.model.metadata.IMetadataTable
|
||||
org.talend.core.model.metadata.IMetadataColumn
|
||||
org.talend.core.model.metadata.IMetadataTable
|
||||
org.talend.core.model.metadata.IMetadataColumn
|
||||
org.talend.core.model.process.IConnection
|
||||
org.talend.core.model.process.IConnectionCategory
|
||||
org.talend.core.model.metadata.types.JavaTypesManager
|
||||
@@ -23,14 +23,14 @@
|
||||
String authMode = ElementParameterParser.getValue(node,"__AUTH_MODE__");
|
||||
String query = ElementParameterParser.getValue(node,"__QUERY__");
|
||||
boolean useLegacySql = ElementParameterParser.getBooleanValue(node,"__USE_LEGACY_SQL__");
|
||||
|
||||
|
||||
query = query.replaceAll("\n"," ").replaceAll("\r"," ");
|
||||
|
||||
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
if (authMode.equals("OAUTH")) {
|
||||
if (authMode.equals("OAUTH") || authMode.equals("TOKEN")) {
|
||||
String resultSizeType = ElementParameterParser.getValue(node,"__RESULT_SIZE__");
|
||||
%>
|
||||
|
||||
|
||||
// Start a Query Job
|
||||
String querySql_<%=cid %> = <%=query %>;
|
||||
System.out.format("Running Query : %s\n", querySql_<%=cid %>);
|
||||
@@ -41,17 +41,20 @@
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Try <%="LARGE".equals(resultSizeType) ? "with" : "without"%> allow large results flag");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
if(authMode.equals("TOKEN") && !ElementParameterParser.canEncrypt(node, "__ACCESS_TOKEN__")) { %>
|
||||
credential_<%=cid%>.setAccessToken(<%= ElementParameterParser.getValue(node, "__ACCESS_TOKEN__")%>);
|
||||
<% } %>
|
||||
|
||||
com.google.api.services.bigquery.model.Job insert_<%=cid %> = bigQueryUtil_<%=cid%>.executeQuery(querySql_<%=cid%>, <%="LARGE".equals(resultSizeType) ? true : false%>, <%=useLegacySql%>);
|
||||
|
||||
<%
|
||||
|
||||
<%
|
||||
if(isLog4jEnabled){
|
||||
%>
|
||||
log.info("<%=cid%> - Executing query.");
|
||||
@@ -88,12 +91,12 @@
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\", \"OAUTH\" or \"TOKEN\", but it is " + authMode);
|
||||
}
|
||||
|
||||
|
||||
List< ? extends IConnection> conns = node.getOutgoingSortedConnections();
|
||||
if (conns != null){
|
||||
if (conns.size()>0){
|
||||
@@ -101,7 +104,7 @@
|
||||
String connName = conn.getName();
|
||||
if (conn.getLineStyle().hasConnectionCategory(IConnectionCategory.DATA)) {
|
||||
|
||||
if (authMode.equals("OAUTH")) {
|
||||
if (authMode.equals("OAUTH") || authMode.equals("TOKEN")) {
|
||||
%>
|
||||
while (true) {
|
||||
// Fetch Results
|
||||
@@ -112,14 +115,14 @@
|
||||
insert_<%=cid %>.getConfiguration().getQuery()
|
||||
.getDestinationTable().getTableId())
|
||||
.setPageToken(pageToken_<%=cid%>).execute();
|
||||
|
||||
|
||||
List<com.google.api.services.bigquery.model.TableRow> rows_<%=cid %> = dataList_<%=cid %>.getRows();
|
||||
|
||||
if(rows_<%=cid %> == null) {
|
||||
// Means there is no record.
|
||||
rows_<%=cid %> = new java.util.ArrayList<com.google.api.services.bigquery.model.TableRow>();
|
||||
}
|
||||
|
||||
|
||||
for (com.google.api.services.bigquery.model.TableRow row_<%=cid %> : rows_<%=cid %>) {
|
||||
java.util.List<com.google.api.services.bigquery.model.TableCell> field_<%=cid %> = row_<%=cid %>.getF();
|
||||
Object value_<%=cid%> = null;
|
||||
@@ -130,15 +133,15 @@
|
||||
com.google.cloud.bigquery.TableResult result_<%=cid%> = job_<%=cid%>.getQueryResults();
|
||||
|
||||
long nb_line_<%=cid%> = 0;
|
||||
|
||||
|
||||
for (com.google.cloud.bigquery.FieldValueList field_<%=cid %> : result_<%=cid%>.iterateAll()) {
|
||||
Object value_<%=cid%>;
|
||||
nb_line_<%=cid%> ++;
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\", \"OAUTH\" or \"TOKEN\", but it is " + authMode);
|
||||
}
|
||||
|
||||
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
if ((metadatas!=null) && (metadatas.size() > 0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
@@ -148,17 +151,17 @@
|
||||
boolean advancedSeparator = (advancedSeparatorStr!=null&&!("").equals(advancedSeparatorStr))?("true").equals(advancedSeparatorStr):false;
|
||||
String thousandsSeparator = ElementParameterParser.getValueWithJavaType(node, "__THOUSANDS_SEPARATOR__", JavaTypesManager.CHARACTER);
|
||||
String decimalSeparator = ElementParameterParser.getValueWithJavaType(node, "__DECIMAL_SEPARATOR__", JavaTypesManager.CHARACTER);
|
||||
|
||||
|
||||
List<IMetadataColumn> columns = metadata.getListColumns();
|
||||
int nbColumns = columns.size();
|
||||
for (int i = 0; i < nbColumns; i++ ) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
String columnName = column.getLabel();
|
||||
|
||||
|
||||
String typeToGenerate = JavaTypesManager.getTypeToGenerate(column.getTalendType(), column.isNullable());
|
||||
JavaType javaType = JavaTypesManager.getJavaTypeFromId(column.getTalendType());
|
||||
String patternValue = column.getPattern() == null || column.getPattern().trim().length() == 0 ? null : column.getPattern();
|
||||
if (authMode.equals("OAUTH")) {
|
||||
String patternValue = column.getPattern() == null || column.getPattern().trim().length() == 0 ? null : column.getPattern();
|
||||
if (authMode.equals("OAUTH") || authMode.equals("TOKEN")) {
|
||||
%>
|
||||
value_<%=cid%> = field_<%=cid %>.get(<%=i%>).getV();
|
||||
<%
|
||||
@@ -167,22 +170,22 @@
|
||||
value_<%=cid%> = field_<%=cid %>.get(<%=i%>).getValue();
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\", \"OAUTH\" or \"TOKEN\", but it is " + authMode);
|
||||
}
|
||||
%>
|
||||
if(com.google.api.client.util.Data.isNull(value_<%=cid%>)) value_<%=cid%> = null;
|
||||
if(value_<%=cid%> != null){
|
||||
|
||||
|
||||
<%
|
||||
if (javaType == JavaTypesManager.STRING) {
|
||||
%>
|
||||
<%=connName%>.<%=columnName%> = value_<%=cid%>.toString();
|
||||
<%
|
||||
} else if (javaType == JavaTypesManager.OBJECT) {
|
||||
%>
|
||||
%>
|
||||
<%=connName%>.<%=columnName%> = value_<%=cid%>;
|
||||
<%
|
||||
} else if(javaType == JavaTypesManager.DATE) {
|
||||
<%
|
||||
} else if(javaType == JavaTypesManager.DATE) {
|
||||
%>
|
||||
if (value_<%=cid%>.toString().contains("-")) {
|
||||
String sValue_<%=cid%> = value_<%=cid%>.toString();
|
||||
@@ -195,13 +198,13 @@
|
||||
} else{
|
||||
<%=connName%>.<%=columnName%>=ParserUtils.parseTo_Date(value_<%=cid%>.toString());
|
||||
}
|
||||
<%
|
||||
} else if(advancedSeparator && JavaTypesManager.isNumberType(javaType)) {
|
||||
<%
|
||||
} else if(advancedSeparator && JavaTypesManager.isNumberType(javaType)) {
|
||||
%>
|
||||
<%=connName%>.<%=columnName%> = ParserUtils.parseTo_<%= typeToGenerate %>(ParserUtils.parseTo_Number(value_<%=cid%>.toString(), <%= thousandsSeparator %>, <%= decimalSeparator %>));
|
||||
<%
|
||||
} else if(javaType == JavaTypesManager.BYTE_ARRAY) {
|
||||
%>
|
||||
} else if(javaType == JavaTypesManager.BYTE_ARRAY) {
|
||||
%>
|
||||
<%=connName%>.<%=columnName%> = value_<%=cid%>.toString().getBytes(<%=encoding %>);
|
||||
<%
|
||||
} else {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
LONG_NAME=Connect and run a query on Google BigQuery
|
||||
LONG_NAME=Connect and run a query on Google BigQuery
|
||||
HELP=org.talend.help.tBigQuerySQLRow
|
||||
|
||||
AUTHENTICATION.NAME=Authentication
|
||||
@@ -6,11 +6,14 @@ SERVICE_ACCOUNT_CREDENTIALS_FILE.NAME=Service account credentials file
|
||||
AUTH_MODE.NAME=Authentication mode
|
||||
AUTH_MODE.ITEM.SERVICEACCOUNT=Service account
|
||||
AUTH_MODE.ITEM.OAUTH=OAuth 2.0
|
||||
AUTH_MODE.ITEM.TOKEN=OAuth Access Token
|
||||
|
||||
CLIENT_ID.NAME=Client Id
|
||||
CLIENT_SECRET.NAME=Client Secret
|
||||
PROJECT_ID.NAME=Project Id
|
||||
PROJECT_ID.NAME=Project ID
|
||||
AUTHORIZATION_CODE.NAME=Authorization Code
|
||||
ACCESS_TOKEN.NAME=OAuth Access Token
|
||||
|
||||
QUERY.NAME=Query
|
||||
SCHEMA.NAME=Schema
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ AUTH_MODE.ITEM.SERVICEACCOUNT=Service account
|
||||
AUTH_MODE.ITEM.OAUTH=OAuth 2.0
|
||||
CLIENT_ID.NAME=Client Id
|
||||
CLIENT_SECRET.NAME=Client Secret
|
||||
PROJECT_ID.NAME=Project Id
|
||||
PROJECT_ID.NAME=Project ID
|
||||
AUTHORIZATION_CODE.NAME=Authorization Code
|
||||
QUERY.NAME=Query
|
||||
SCHEMA.NAME=Schema
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<%@ jet
|
||||
<%@ jet
|
||||
imports="
|
||||
org.talend.core.model.process.INode
|
||||
org.talend.designer.codegen.config.CodeGeneratorArgument
|
||||
@@ -6,7 +6,7 @@ imports="
|
||||
org.talend.core.model.process.IConnection
|
||||
org.talend.core.model.process.IConnectionCategory
|
||||
java.util.List
|
||||
"
|
||||
"
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/Log4j/Log4jFileUtil.javajet"%>
|
||||
<%
|
||||
@@ -20,7 +20,7 @@ imports="
|
||||
IConnection conn =conns.get(0);
|
||||
String connName = conn.getName();
|
||||
if (conn.getLineStyle().hasConnectionCategory(IConnectionCategory.DATA)) {
|
||||
if (authMode.equals("OAUTH")) {
|
||||
if (authMode.equals("OAUTH") || authMode.equals("TOKEN")) {
|
||||
%>
|
||||
}
|
||||
pageToken_<%=cid%> = dataList_<%=cid %>.getPageToken();
|
||||
@@ -35,7 +35,7 @@ imports="
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\", but it is " + authMode);
|
||||
throw new IllegalArgumentException("authentication mode should be either \"SERVICEACCOUNT\" or \"OAUTH\"or \"TOKEN\", but it is " + authMode);
|
||||
}
|
||||
log4jFileUtil.retrievedDataNumberInfo(node);
|
||||
}
|
||||
|
||||
@@ -165,7 +165,9 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="xstream-1.4.10.jar" MODULE="xstream-1.4.10.jar" MVN="mvn:com.thoughtworks.xstream/xstream/1.4.10" REQUIRED_IF="(DB_VERSION=='BONITA_523') OR (((DB_VERSION=='BONITA_652')OR(DB_VERSION=='BONITA_724')) AND (CLIENT_MODE=='JAVA_CLIENT'))" />
|
||||
|
||||
<IMPORT NAME="xstream-1.4.15.jar" MODULE="xstream-1.4.15.jar" MVN="mvn:com.thoughtworks.xstream/xstream/1.4.15" REQUIRED_IF="(DB_VERSION=='BONITA_523') OR (((DB_VERSION=='BONITA_652')OR(DB_VERSION=='BONITA_724')) AND (CLIENT_MODE=='JAVA_CLIENT'))" />
|
||||
|
||||
<!-- Bonita 5.2.3 -->
|
||||
<IMPORT NAME="bonita_client_523" MODULE="bonita-client-5.2.3.jar" MVN="mvn:org.talend.libraries/bonita-client-5.2.3/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.bonita/lib/bonita-client-5.2.3.jar" REQUIRED_IF="DB_VERSION=='BONITA_523'" />
|
||||
<IMPORT NAME="bonita_server_523" MODULE="bonita-server-5.2.3.jar" MVN="mvn:org.talend.libraries/bonita-server-5.2.3/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.bonita/lib/bonita-server-5.2.3.jar" REQUIRED_IF="DB_VERSION=='BONITA_523'" />
|
||||
@@ -181,7 +183,7 @@
|
||||
<IMPORT NAME="ehcache_123" MODULE="ehcache-1.2.3.jar" MVN="mvn:org.talend.libraries/ehcache-1.2.3/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.ehcache/lib/ehcache-1.2.3.jar" REQUIRED="false" />
|
||||
<IMPORT NAME="groovy_all_170" MODULE="groovy-all-1.7.0.jar" MVN="mvn:org.talend.libraries/groovy-all-1.7.0/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.groovy/lib/groovy-all-1.7.0.jar" REQUIRED="false" />
|
||||
|
||||
<IMPORT NAME="h2_12132" MODULE="h2-1.2.132.jar" MVN="mvn:org.talend.libraries/h2-1.2.132/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.h2/lib/h2-1.2.132.jar" REQUIRED="false" />
|
||||
<IMPORT NAME="h2_12132" MODULE="h2-1.2.132.jar" MVN="mvn:com.h2database/h2/1.2.132" REQUIRED="false" />
|
||||
<IMPORT NAME="hibernate_326_ga" MODULE="hibernate-3.2.6.ga.jar" MVN="mvn:org.talend.libraries/hibernate-3.2.6.ga/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.hibernate/lib/hibernate-3.2.6.ga.jar" REQUIRED="false" />
|
||||
<IMPORT NAME="jline_0994" MODULE="jline-0.9.94.jar" MVN="mvn:org.talend.libraries/jline-0.9.94/6.0.0" REQUIRED="false" />
|
||||
<IMPORT NAME="jta_11" MODULE="jta-1.1.jar" MVN="mvn:javax.transaction/jta/1.1" REQUIRED_IF="DB_VERSION=='BONITA_561' OR DB_VERSION=='BONITA_5101'" />
|
||||
@@ -208,20 +210,20 @@
|
||||
<IMPORT NAME="bonita_server_561" MODULE="bonita-server-5.6.1.jar" MVN="mvn:org.talend.libraries/bonita-server-5.6.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.bonita/lib/bonita-server-5.6.1.jar" REQUIRED_IF="DB_VERSION=='BONITA_561'" />
|
||||
<IMPORT NAME="chemistry_opencmis_client_api_020_incubating" MODULE="chemistry-opencmis-client-api-0.2.0-incubating.jar" MVN="mvn:org.talend.libraries/chemistry-opencmis-client-api-0.2.0-incubating/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.chemistry/lib/chemistry-opencmis-client-api-0.2.0-incubating.jar" REQUIRED_IF="DB_VERSION=='BONITA_561'" />
|
||||
<IMPORT NAME="chemistry_opencmis_commons_api_020_incubating" MODULE="chemistry-opencmis-commons-api-0.2.0-incubating.jar" MVN="mvn:org.talend.libraries/chemistry-opencmis-commons-api-0.2.0-incubating/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.chemistry/lib/chemistry-opencmis-commons-api-0.2.0-incubating.jar" REQUIRED_IF="DB_VERSION=='BONITA_561'" />
|
||||
<IMPORT NAME="h2_12139" MODULE="h2-1.2.139.jar" MVN="mvn:org.talend.libraries/h2-1.2.139/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.h2/lib/h2-1.2.139.jar" REQUIRED_IF="DB_VERSION=='BONITA_561' OR DB_VERSION=='BONITA_5101'" />
|
||||
<IMPORT NAME="h2_12139" MODULE="h2-1.2.139.jar" MVN="mvn:com.h2database/h2/1.2.139" REQUIRED_IF="DB_VERSION=='BONITA_561' OR DB_VERSION=='BONITA_5101'" />
|
||||
<IMPORT NAME="slf4j_api_161" MODULE="slf4j-api-1.6.1.jar" MVN="mvn:org.talend.libraries/slf4j-api-1.6.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.slf4j/lib/slf4j-api-1.6.1.jar" REQUIRED_IF="DB_VERSION=='BONITA_561' OR DB_VERSION=='BONITA_5101'" />
|
||||
<IMPORT NAME="slf4j_jdk14_161" MODULE="slf4j-jdk14-1.6.1.jar" MVN="mvn:org.talend.libraries/slf4j-jdk14-1.6.1/6.0.0" REQUIRED_IF="DB_VERSION=='BONITA_561' OR DB_VERSION=='BONITA_5101'" />
|
||||
|
||||
<!-- Bonita 5.10.1 -->
|
||||
<IMPORT NAME="bonita_server_5101" MODULE="bonita-server-5.10.1.jar" MVN="mvn:org.talend.libraries/bonita-server-5.10.1/6.0.0" REQUIRED_IF="DB_VERSION=='BONITA_5101'" />
|
||||
<IMPORT NAME="hibernate_search_330_Final" MODULE="hibernate-search-3.3.0.Final.jar" MVN="mvn:org.talend.libraries/hibernate-search-3.3.0.Final/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.hibernate/lib/hibernate-search-3.3.0.Final.jar" REQUIRED_IF="DB_VERSION=='BONITA_5101'" />
|
||||
<IMPORT NAME="hibernate_core_3_6_10_Final" MODULE="hibernate-core-3.6.10.Final.jar" MVN="mvn:org.talend.libraries/hibernate-core-3.6.10.Final/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.hibernate/lib/hibernate-core-3.6.10.Final.jar" REQUIRED_IF="DB_VERSION=='BONITA_5101'" />
|
||||
<IMPORT NAME="hibernate_core_3_6_10_Final" MODULE="hibernate-core-3.6.10.Final.jar" MVN="mvn:org.hibernate/hibernate-core/3.6.10.Final" REQUIRED_IF="DB_VERSION=='BONITA_5101'" />
|
||||
<IMPORT NAME="hibernate_jpa_2_0_api-1_0_1_Final" MODULE="hibernate-jpa-2.0-api-1.0.1.Final.jar" MVN="mvn:org.talend.libraries/hibernate-jpa-2.0-api-1.0.1.Final/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.hibernate/lib/hibernate-jpa-2.0-api-1.0.1.Final.jar" REQUIRED_IF="DB_VERSION=='BONITA_5101'" />
|
||||
<IMPORT NAME="lucene_core_303" MODULE="lucene-core-3.0.3.jar" MVN="mvn:org.talend.libraries/lucene-core-3.0.3/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.lucene/lib/lucene-core-3.0.3.jar" REQUIRED_IF="DB_VERSION=='BONITA_5101'" />
|
||||
<IMPORT NAME="javassist_3120_GA" MODULE="javassist-3.12.0.GA.jar" MVN="mvn:org.talend.libraries/javassist-3.12.0.GA/6.0.0" REQUIRED_IF="DB_VERSION=='BONITA_5101'" />
|
||||
|
||||
<!-- Bonita 6.5.2 HTTP API -->
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client-1.0.0/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client/1.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="bonita-client-6.5.2.jar" MODULE="bonita-client-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-client-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="bonita-common-6.5.2.jar" MODULE="bonita-common-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-common-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
|
||||
@@ -232,7 +234,7 @@
|
||||
<IMPORT NAME="commons-logging-1.2.jar" MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
|
||||
<!-- Bonita 6.5.2 JAVA API -->
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client-1.0.0/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client/1.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="bonita-client-6.5.2.jar" MODULE="bonita-client-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-client-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="bonita-common-6.5.2.jar" MODULE="bonita-common-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-common-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="bonita-server-6.5.2.jar" MODULE="bonita-server-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-server-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
@@ -245,7 +247,7 @@
|
||||
<IMPORT NAME="commons-codec-1.14.jar" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
|
||||
<!-- Bonita 7.2.4 HTTP API -->
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client-1.0.0/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client/1.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="bonita-client-7.2.4.jar" MODULE="bonita-client-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-client-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="bonita-common-7.2.4.jar" MODULE="bonita-common-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-common-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
|
||||
@@ -256,7 +258,7 @@
|
||||
<IMPORT NAME="commons-codec-1.14.jar" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
|
||||
<!-- Bonita 7.2.4 JAVA API -->
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client-1.0.0/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client/1.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="bonita-client-7.2.4.jar" MODULE="bonita-client-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-client-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="bonita-common-7.2.4.jar" MODULE="bonita-common-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-common-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="bonita-server-7.2.4.jar" MODULE="bonita-server-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-server-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
|
||||
@@ -214,7 +214,8 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="xstream-1.4.10.jar" MODULE="xstream-1.4.10.jar" MVN="mvn:com.thoughtworks.xstream/xstream/1.4.10" REQUIRED_IF="((DB_VERSION!='BONITA_652') AND (DB_VERSION!='BONITA_724')) OR (((DB_VERSION=='BONITA_652')OR(DB_VERSION=='BONITA_724')) AND (CLIENT_MODE=='JAVA_CLIENT'))" />
|
||||
|
||||
<IMPORT NAME="xstream-1.4.15.jar" MODULE="xstream-1.4.15.jar" MVN="mvn:com.thoughtworks.xstream/xstream/1.4.15" REQUIRED_IF="((DB_VERSION!='BONITA_652') AND (DB_VERSION!='BONITA_724')) OR (((DB_VERSION=='BONITA_652')OR(DB_VERSION=='BONITA_724')) AND (CLIENT_MODE=='JAVA_CLIENT'))" />
|
||||
<!-- Bonita 5.2.3 -->
|
||||
<IMPORT NAME="bonita_client_523" MODULE="bonita-client-5.2.3.jar" MVN="mvn:org.talend.libraries/bonita-client-5.2.3/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.bonita/lib/bonita-client-5.2.3.jar" REQUIRED_IF="DB_VERSION=='BONITA_523'" />
|
||||
<IMPORT NAME="bonita_server_523" MODULE="bonita-server-5.2.3.jar" MVN="mvn:org.talend.libraries/bonita-server-5.2.3/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.bonita/lib/bonita-server-5.2.3.jar" REQUIRED_IF="DB_VERSION=='BONITA_523'" />
|
||||
@@ -230,7 +231,7 @@
|
||||
<IMPORT NAME="ehcache_123" MODULE="ehcache-1.2.3.jar" MVN="mvn:org.talend.libraries/ehcache-1.2.3/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.ehcache/lib/ehcache-1.2.3.jar" REQUIRED="false" />
|
||||
<IMPORT NAME="groovy_all_170" MODULE="groovy-all-1.7.0.jar" MVN="mvn:org.talend.libraries/groovy-all-1.7.0/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.groovy/lib/groovy-all-1.7.0.jar" REQUIRED="false" />
|
||||
|
||||
<IMPORT NAME="h2_12132" MODULE="h2-1.2.132.jar" MVN="mvn:org.talend.libraries/h2-1.2.132/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.h2/lib/h2-1.2.132.jar" REQUIRED="false" />
|
||||
<IMPORT NAME="h2_12132" MODULE="h2-1.2.132.jar" MVN="mvn:com.h2database/h2/1.2.132" REQUIRED="false" />
|
||||
<IMPORT NAME="hibernate_326_ga" MODULE="hibernate-3.2.6.ga.jar" MVN="mvn:org.talend.libraries/hibernate-3.2.6.ga/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.hibernate/lib/hibernate-3.2.6.ga.jar" REQUIRED="false" />
|
||||
<IMPORT NAME="jline_0994" MODULE="jline-0.9.94.jar" MVN="mvn:org.talend.libraries/jline-0.9.94/6.0.0" REQUIRED="false" />
|
||||
<IMPORT NAME="jta_11" MODULE="jta-1.1.jar" MVN="mvn:javax.transaction/jta/1.1" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_561' OR DB_VERSION=='BONITA_5101')" />
|
||||
@@ -257,19 +258,19 @@
|
||||
<IMPORT NAME="bonita_server_561" MODULE="bonita-server-5.6.1.jar" MVN="mvn:org.talend.libraries/bonita-server-5.6.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.bonita/lib/bonita-server-5.6.1.jar" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_561')" />
|
||||
<IMPORT NAME="chemistry_opencmis_client_api_020_incubating" MODULE="chemistry-opencmis-client-api-0.2.0-incubating.jar" MVN="mvn:org.talend.libraries/chemistry-opencmis-client-api-0.2.0-incubating/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.chemistry/lib/chemistry-opencmis-client-api-0.2.0-incubating.jar" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_561')" />
|
||||
<IMPORT NAME="chemistry_opencmis_commons_api_020_incubating" MODULE="chemistry-opencmis-commons-api-0.2.0-incubating.jar" MVN="mvn:org.talend.libraries/chemistry-opencmis-commons-api-0.2.0-incubating/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.chemistry/lib/chemistry-opencmis-commons-api-0.2.0-incubating.jar" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_561')" />
|
||||
<IMPORT NAME="h2_12139" MODULE="h2-1.2.139.jar" MVN="mvn:org.talend.libraries/h2-1.2.139/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.h2/lib/h2-1.2.139.jar" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_561' OR DB_VERSION=='BONITA_5101')" />
|
||||
<IMPORT NAME="h2_12139" MODULE="h2-1.2.139.jar" MVN="mvn:com.h2database/h2/1.2.139" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_561' OR DB_VERSION=='BONITA_5101')" />
|
||||
<IMPORT NAME="slf4j_api_161" MODULE="slf4j-api-1.6.1.jar" MVN="mvn:org.talend.libraries/slf4j-api-1.6.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.slf4j/lib/slf4j-api-1.6.1.jar" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_561' OR DB_VERSION=='BONITA_5101')" />
|
||||
<IMPORT NAME="slf4j_jdk14_161" MODULE="slf4j-jdk14-1.6.1.jar" MVN="mvn:org.talend.libraries/slf4j-jdk14-1.6.1/6.0.0" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_561' OR DB_VERSION=='BONITA_5101')" />
|
||||
<!-- Bonita 5.10.1 -->
|
||||
<IMPORT NAME="bonita_server_510" MODULE="bonita-server-5.10.1.jar" MVN="mvn:org.talend.libraries/bonita-server-5.10.1/6.0.0" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_5101')" />
|
||||
<IMPORT NAME="hibernate_search_330_Final" MODULE="hibernate-search-3.3.0.Final.jar" MVN="mvn:org.talend.libraries/hibernate-search-3.3.0.Final/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.hibernate/lib/hibernate-search-3.3.0.Final.jar" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_5101')" />
|
||||
<IMPORT NAME="hibernate_core_3_6_10_Final" MODULE="hibernate-core-3.6.10.Final.jar" MVN="mvn:org.talend.libraries/hibernate-core-3.6.10.Final/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.hibernate/lib/hibernate-core-3.6.10.Final.jar" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_5101')" />
|
||||
<IMPORT NAME="hibernate_core_3_6_10_Final" MODULE="hibernate-core-3.6.10.Final.jar" MVN="mvn:org.hibernate/hibernate-core/3.6.10.Final" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_5101')" />
|
||||
<IMPORT NAME="hibernate_jpa_2_0_api-1_0_1_Final" MODULE="hibernate-jpa-2.0-api-1.0.1.Final.jar" MVN="mvn:org.talend.libraries/hibernate-jpa-2.0-api-1.0.1.Final/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.hibernate/lib/hibernate-jpa-2.0-api-1.0.1.Final.jar" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_5101')" />
|
||||
<IMPORT NAME="lucene_core_303" MODULE="lucene-core-3.0.3.jar" MVN="mvn:org.talend.libraries/lucene-core-3.0.3/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.lucene/lib/lucene-core-3.0.3.jar" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_5101')" />
|
||||
<IMPORT NAME="javassist_3120_GA" MODULE="javassist-3.12.0.GA.jar" MVN="mvn:org.talend.libraries/javassist-3.12.0.GA/6.0.0" REQUIRED_IF="(CLIENT_MODE=='JAVA_CLIENT') AND (DB_VERSION=='BONITA_5101')" />
|
||||
|
||||
<!-- Bonita 6.5.2 HTTP API -->
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client-1.0.0/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client/1.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="bonita-client-6.5.2.jar" MODULE="bonita-client-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-client-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="bonita-common-6.5.2.jar" MODULE="bonita-common-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-common-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
|
||||
@@ -281,7 +282,7 @@
|
||||
<IMPORT NAME="commons-lang3-3.8.1.jar" MODULE="commons-lang3-3.8.1.jar" MVN="mvn:org.apache.commons/commons-lang3/3.8.1" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang3-3.8.1.jar" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
|
||||
<!-- Bonita 6.5.2 JAVA API -->
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client-1.0.0/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client/1.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="bonita-client-6.5.2.jar" MODULE="bonita-client-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-client-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="bonita-common-6.5.2.jar" MODULE="bonita-common-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-common-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="bonita-server-6.5.2.jar" MODULE="bonita-server-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-server-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
@@ -294,7 +295,7 @@
|
||||
<IMPORT NAME="commons-codec-1.14.jar" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
|
||||
<!-- Bonita 7.2.4 HTTP API -->
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client-1.0.0/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client/1.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="bonita-client-7.2.4.jar" MODULE="bonita-client-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-client-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="bonita-common-7.2.4.jar" MODULE="bonita-common-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-common-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
|
||||
@@ -306,7 +307,7 @@
|
||||
<IMPORT NAME="commons-lang3-3.8.1.jar" MODULE="commons-lang3-3.8.1.jar" MVN="mvn:org.apache.commons/commons-lang3/3.8.1" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang3-3.8.1.jar" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
|
||||
<!-- Bonita 7.2.4 JAVA API -->
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client-1.0.0/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="talend-bonita-client-1.0.0.jar" MODULE="talend-bonita-client-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-bonita-client/1.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="bonita-client-7.2.4.jar" MODULE="bonita-client-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-client-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="bonita-common-7.2.4.jar" MODULE="bonita-common-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-common-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="bonita-server-7.2.4.jar" MODULE="bonita-server-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-server-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
|
||||
@@ -72,13 +72,13 @@
|
||||
MODULE="boxjavalibv2-v3.0.10.jar" MVN="mvn:net.box/boxjavalibv2/v3.0.10"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core"
|
||||
MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
MODULE="jackson-core-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind"
|
||||
MODULE="jackson-databind-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
MODULE="jackson-databind-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations"
|
||||
MODULE="jackson-annotations-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
MODULE="jackson-annotations-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="apache-http-client"
|
||||
MODULE="httpclient-4.5.12.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.12"
|
||||
|
||||
@@ -127,13 +127,13 @@
|
||||
MODULE="boxjavalibv2-v3.0.10.jar" MVN="mvn:net.box/boxjavalibv2/v3.0.10"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-core"
|
||||
MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
MODULE="jackson-core-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-databind"
|
||||
MODULE="jackson-databind-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
MODULE="jackson-databind-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-annotations"
|
||||
MODULE="jackson-annotations-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
MODULE="jackson-annotations-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-client"
|
||||
MODULE="httpclient-4.5.12.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.12"
|
||||
|
||||
@@ -88,13 +88,13 @@
|
||||
MODULE="boxjavalibv2-v3.0.10.jar" MVN="mvn:net.box/boxjavalibv2/v3.0.10"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-core"
|
||||
MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
MODULE="jackson-core-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-databind"
|
||||
MODULE="jackson-databind-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
MODULE="jackson-databind-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-annotations"
|
||||
MODULE="jackson-annotations-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
MODULE="jackson-annotations-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-client"
|
||||
MODULE="httpclient-4.5.12.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.12"
|
||||
|
||||
@@ -98,13 +98,13 @@
|
||||
MODULE="boxjavalibv2-v3.0.10.jar" MVN="mvn:net.box/boxjavalibv2/v3.0.10"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-core"
|
||||
MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
MODULE="jackson-core-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-databind"
|
||||
MODULE="jackson-databind-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
MODULE="jackson-databind-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-annotations"
|
||||
MODULE="jackson-annotations-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
MODULE="jackson-annotations-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-client"
|
||||
MODULE="httpclient-4.5.12.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.12"
|
||||
|
||||
@@ -106,13 +106,13 @@
|
||||
MODULE="boxjavalibv2-v3.0.10.jar" MVN="mvn:net.box/boxjavalibv2/v3.0.10"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-core"
|
||||
MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
MODULE="jackson-core-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-databind"
|
||||
MODULE="jackson-databind-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
MODULE="jackson-databind-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-annotations"
|
||||
MODULE="jackson-annotations-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
MODULE="jackson-annotations-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-client"
|
||||
MODULE="httpclient-4.5.12.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.12"
|
||||
|
||||
@@ -109,13 +109,13 @@
|
||||
MODULE="boxjavalibv2-v3.0.10.jar" MVN="mvn:net.box/boxjavalibv2/v3.0.10"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-core"
|
||||
MODULE="jackson-core-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
MODULE="jackson-core-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-databind"
|
||||
MODULE="jackson-databind-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
MODULE="jackson-databind-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="jackson-annotations"
|
||||
MODULE="jackson-annotations-2.10.1.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
MODULE="jackson-annotations-2.11.4.jar" MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-client"
|
||||
MODULE="httpclient-4.5.12.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.12"
|
||||
|
||||
@@ -56,7 +56,10 @@ COMPATIBILITY="ALL"
|
||||
REQUIRED="true">
|
||||
<DEFAULT>"ISO-8859-15"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="BUFFERSIZE" FIELD="TEXT" REQUIRED="true"
|
||||
NUM_ROW="50">
|
||||
<DEFAULT>8192</DEFAULT>
|
||||
</PARAMETER>
|
||||
</PARAMETERS>
|
||||
|
||||
<ADVANCED_PARAMETERS>
|
||||
|
||||
@@ -19,6 +19,7 @@ String sOutEncoding = ElementParameterParser.getValue(node, "__ENCODING__");
|
||||
boolean bNeedEncode = "true".equals(ElementParameterParser.getValue(node, "__USE_INENCODING__"));
|
||||
boolean bCreateDir = "true".equals(ElementParameterParser.getValue(node, "__CREATE__"));
|
||||
String sTmpOutFileName = outFileName;
|
||||
final String bufferSizeExpression = ElementParameterParser.getValue(node, "__BUFFERSIZE__");
|
||||
|
||||
log4jFileUtil.componentStartInfo(node);
|
||||
if (outFileName.equals(inFileName)){
|
||||
@@ -37,15 +38,23 @@ java.io.File outFile_<%=cid%> = new java.io.File(sOutFileN_<%=cid%>);
|
||||
parentFile_<%=cid%>.mkdirs();
|
||||
}
|
||||
<%}%>
|
||||
|
||||
Integer bufferSize_<%=cid%> = <%= bufferSizeExpression %>;
|
||||
if (bufferSize_<%=cid%> == null) {
|
||||
bufferSize_<%=cid%> = 8192;
|
||||
}
|
||||
|
||||
final java.io.BufferedWriter out<%=cid %> = new java.io.BufferedWriter(
|
||||
new java.io.OutputStreamWriter(
|
||||
new java.io.FileOutputStream(sOutFileN_<%=cid%>, false), <%=sOutEncoding%>
|
||||
));
|
||||
),
|
||||
bufferSize_<%=cid%>);
|
||||
|
||||
java.io.BufferedReader in_<%=cid %> = new java.io.BufferedReader(
|
||||
new java.io.InputStreamReader(
|
||||
new java.io.FileInputStream(sOriFileN_<%=cid%>), <%=bNeedEncode ? sInEncoding : "System.getProperty(\"file.encoding\")"%>
|
||||
));
|
||||
),
|
||||
bufferSize_<%=cid%>);
|
||||
|
||||
//8192: the default buffer size of BufferedReader
|
||||
char[] cbuf_<%=cid %> = new char[8192];
|
||||
|
||||
@@ -10,3 +10,4 @@ INENCODING.NAME=Input Encoding
|
||||
INFILE_NAME.NAME=Input File Name
|
||||
OUTFILE_NAME.NAME=Output File Name
|
||||
CREATE.NAME=Create directory if not exists
|
||||
BUFFERSIZE.NAME=Buffer Size
|
||||
|
||||
@@ -165,7 +165,6 @@ COMPATIBILITY="ALL"
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="aopalliance-1_0" MODULE="aopalliance-1.0.jar" MVN="mvn:aopalliance/aopalliance/1.0" UrlPath="platform:/plugin/org.talend.libraries.esb/lib/aopalliance-1.0.jar" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="aws-ec2-2_0_2" MODULE="aws-ec2-2.0.2.jar" MVN="mvn:org.talend.libraries/aws-ec2-2.0.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="bcprov-jdk16-1_46" MODULE="bcprov-jdk16-1.46.jar" MVN="mvn:org.talend.libraries/bcprov-jdk16-1.46/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="commons-io-2_4" MODULE="commons-io-2.4.jar" MVN="mvn:commons-io/commons-io/2.4" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.4.jar" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="ec2-2_0_2" MODULE="ec2-2.0.2.jar" MVN="mvn:org.talend.libraries/ec2-2.0.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="gson-2_2" MODULE="gson-2.2.jar" MVN="mvn:org.talend.libraries/gson-2.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
|
||||
@@ -74,7 +74,6 @@ COMPATIBILITY="ALL"
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="aopalliance-1_0" MODULE="aopalliance-1.0.jar" MVN="mvn:aopalliance/aopalliance/1.0" UrlPath="platform:/plugin/org.talend.libraries.esb/lib/aopalliance-1.0.jar" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="aws-ec2-2_0_2" MODULE="aws-ec2-2.0.2.jar" MVN="mvn:org.talend.libraries/aws-ec2-2.0.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="bcprov-jdk16-1_46" MODULE="bcprov-jdk16-1.46.jar" MVN="mvn:org.talend.libraries/bcprov-jdk16-1.46/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="commons-io-2_4" MODULE="commons-io-2.4.jar" MVN="mvn:commons-io/commons-io/2.4" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.4.jar" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="ec2-2_0_2" MODULE="ec2-2.0.2.jar" MVN="mvn:org.talend.libraries/ec2-2.0.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="gson-2_2" MODULE="gson-2.2.jar" MVN="mvn:org.talend.libraries/gson-2.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
|
||||
@@ -606,11 +606,12 @@
|
||||
SHOW_IF="(DBTYPE=='SYBASE') and (USE_EXISTING_CONNECTION == 'true')"
|
||||
/>
|
||||
|
||||
<PARAMETER NAME="DB_SYBASE_VERSION" FIELD="CLOSED_LIST" NUM_ROW="50"
|
||||
<PARAMETER NAME="DB_SYBASE_VERSION" FIELD="CLOSED_LIST" NUM_ROW="50" REPOSITORY_VALUE="DB_VERSION"
|
||||
SHOW_IF="(USE_EXISTING_CONNECTION == 'false')AND(DBTYPE=='SYBASE')">
|
||||
<ITEMS DEFAULT="SYBSEIQ_16">
|
||||
<ITEM NAME="SYBSEIQ_12_15" VALUE="SYBSEIQ_12_15" />
|
||||
<ITEM NAME="SYBSEIQ_16" VALUE="SYBSEIQ_16" />
|
||||
<ITEM NAME="SYBSEIQ_16_SA" VALUE="SYBSEIQ_16_SA" />
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
<!--Sybase configuration end-->
|
||||
@@ -877,7 +878,7 @@
|
||||
<IMPORT NAME="Driver-Oracle12c" MODULE="ojdbc7.jar" MVN="mvn:org.talend.libraries/ojdbc7/6.0.0" REQUIRED_IF="(DBTYPE=='DBORACLE') AND (DB_VERSION == 'ORACLE_12') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="ORACLE_18" MODULE="ojdbc8-19.3.0.0.jar" MVN="mvn:com.oracle.ojdbc/ojdbc8/19.3.0.0" REQUIRED_IF="(DBTYPE=='DBORACLE') AND (DB_VERSION == 'ORACLE_18') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-POSTGRESQL" MODULE="postgresql-8.4-703.jdbc4.jar" MVN="mvn:postgresql/postgresql/8.4-703.jdbc4" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND ((((DBTYPE=='POSTGRE') or (DBTYPE=='POSTGREPLUS')) AND (DB_POSTGRE_VERSION =='PRIOR_TO_V9')) or (DBTYPE=='GREENPLUM'))" />
|
||||
<IMPORT NAME="Driver-Postgres9" MODULE="postgresql-42.2.9.jar" MVN="mvn:org.postgresql/postgresql/42.2.9" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (((DBTYPE=='POSTGRE') or (DBTYPE=='POSTGREPLUS')) AND (DB_POSTGRE_VERSION =='V9_X'))" />
|
||||
<IMPORT NAME="Driver-Postgres9" MODULE="postgresql-42.2.14.jar" MVN="mvn:org.postgresql/postgresql/42.2.14" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (((DBTYPE=='POSTGRE') or (DBTYPE=='POSTGREPLUS')) AND (DB_POSTGRE_VERSION =='V9_X'))" />
|
||||
<IMPORT NAME="Driver-FIREBIRD" MODULE="jaybird-full-2.1.1.jar" MVN="mvn:org.talend.libraries/jaybird-full-2.1.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.firebird/lib/jaybird-full-2.1.1.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='FIREBIRD')" />
|
||||
<IMPORT NAME="Driver-HSQLDb" MODULE="hsqldb.jar" MVN="mvn:org.talend.libraries/hsqldb/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.hsql/lib/hsqldb.jar" REQUIRED_IF="DBTYPE=='HSQLDB'"/>
|
||||
<IMPORT NAME="Driver-INFORMIX-JDBC" MODULE="ifxjdbc.jar" MVN="mvn:org.talend.libraries/ifxjdbc/6.0.0" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='INFORMIX')" />
|
||||
@@ -894,6 +895,7 @@
|
||||
<IMPORT NAME="Driver-SQLITE-JDBC-NESTED" MODULE="sqlitejdbc-v056.jar" MVN="mvn:org.talend.libraries/sqlitejdbc-v056/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.sqlite3/lib/sqlitejdbc-v056.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='SQLITE')" />
|
||||
<IMPORT NAME="Driver-SYBASE-JCONN3" MODULE="jconn3.jar" MVN="mvn:org.talend.libraries/jconn3/6.0.0" BundleID="" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='SYBASE') AND (DB_SYBASE_VERSION == 'SYBSEIQ_12_15')" />
|
||||
<IMPORT NAME="Driver-SYBASE-JCONN4" MODULE="jconn4.jar" MVN="mvn:org.talend.libraries/jconn4/6.0.0" BundleID="" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='SYBASE') AND (DB_SYBASE_VERSION == 'SYBSEIQ_16')" />
|
||||
<IMPORT NAME="Driver-SqlAnywhere" MODULE="sajdbc4.jar" MVN="mvn:sap.jdbc4.sqlanywhere/sajdbc4/17.0.0" BundleID="" REQUIRED_IF = "(DB_VERSION == 'SYBSEIQ_16_SA') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-Teradata_jdbc" MODULE="terajdbc4-16.20.00.02.jar" MVN="mvn:com.teradata/terajdbc4/16.20.00.02" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='TERADATA')" />
|
||||
<IMPORT NAME="Driver-Teradata_config" MODULE="tdgssconfig-16.20.00.02.jar" MVN="mvn:com.teradata/tdgssconfig/16.20.00.02" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='TERADATA')" />
|
||||
<IMPORT NAME="Driver-Netezza" MODULE="nzjdbc.jar" MVN="mvn:org.talend.libraries/nzjdbc/6.0.0" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='NETEZZA')" />
|
||||
|
||||
@@ -159,6 +159,7 @@ MSSQL_DRIVER.ITEM.MSSQL_PROP=Microsoft
|
||||
DB_SYBASE_VERSION.NAME=DB Version
|
||||
DB_SYBASE_VERSION.ITEM.SYBSEIQ_12_15=Sybase 12/15
|
||||
DB_SYBASE_VERSION.ITEM.SYBSEIQ_16=Sybase 16
|
||||
DB_SYBASE_VERSION.ITEM.SYBSEIQ_16_SA=Sybase 16 (SQL Anywhere)
|
||||
|
||||
ACCOUNT.NAME=Account
|
||||
ROLE.NAME=Role
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user