Compare commits
1 Commits
patch/TPS-
...
allDeltaAn
| Author | SHA1 | Date | |
|---|---|---|---|
|
8b7ef5e95c |
@@ -1,57 +0,0 @@
|
||||
---
|
||||
version: 7.3.1
|
||||
module: https://talend.poolparty.biz/coretaxonomy/42
|
||||
product:
|
||||
- https://talend.poolparty.biz/coretaxonomy/23
|
||||
---
|
||||
|
||||
# TPS-4723
|
||||
|
||||
| Info | Value |
|
||||
| ---------------- | ---------------- |
|
||||
| Patch Name | Patch\_20210304\_TPS-4723\_v1-7.3.1 |
|
||||
| Release Date | 2021-03-04 |
|
||||
| Target Version | 20200219_1130-V7.3.1 |
|
||||
| Product affected | Talend Studio |
|
||||
|
||||
## Introduction
|
||||
|
||||
This is a self-contained patch.
|
||||
|
||||
**NOTE**: For information on how to obtain this patch, reach out to your Support contact at Talend.
|
||||
|
||||
## Fixed issues
|
||||
|
||||
This patch contains the following fixes:
|
||||
|
||||
- TPS-4723 [7.3.1] Getting Permission denied error in tFileInputExcel in Talend (TDI-45561)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Consider the following requirements for your system:
|
||||
|
||||
- Talend Studio 7.3.1 and Patch_20200529_R2020-05_v1-7.3.1 must be installed.
|
||||
|
||||
## Installation
|
||||
|
||||
### Installing the patch using Software update
|
||||
|
||||
1) Logon TAC and switch to Configuration->Software Update, then enter the correct values and save referring to the documentation: https://help.talend.com/reader/f7Em9WV_cPm2RRywucSN0Q/j9x5iXV~vyxMlUafnDejaQ
|
||||
|
||||
2) Switch to Software update page, where the new patch will be listed. The patch can be downloaded from here into the nexus repository.
|
||||
|
||||
3) On Studio Side: Logon Studio with remote mode, on the logon page the Update button is displayed: click this button to install the patch.
|
||||
|
||||
### Installing the patch using Talend Studio
|
||||
|
||||
1) Create a folder named "patches" under your studio installer directory and copy the patch .zip file to this folder.
|
||||
|
||||
2) Restart your studio: a window pops up, then click OK to install the patch, or restart the commandline and the patch will be installed automatically.
|
||||
|
||||
### Installing the patch using Commandline
|
||||
|
||||
Execute the following commands:
|
||||
|
||||
1. Talend-Studio-win-x86_64.exe -nosplash -application org.talend.commandline.CommandLine -consoleLog -data commandline-workspace startServer -p 8002 --talendDebug
|
||||
2. initRemote {tac_url} -ul {TAC login username} -up {TAC login password}
|
||||
3. checkAndUpdate -tu {TAC login username} -tup {TAC login password}
|
||||
@@ -16,9 +16,11 @@
|
||||
</requires>
|
||||
<plugin id="org.talend.libraries.apache" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.axis2" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.batik" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.chemistry" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.common" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.cxf" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.google" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.http" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.lucene" download-size="0" install-size="0" version="0.0.0"/>
|
||||
<plugin id="org.talend.libraries.apache.xml" download-size="0" install-size="0" version="0.0.0"/>
|
||||
|
||||
@@ -50,11 +50,8 @@ List<IConnection> allSubProcessConnection = codeGenArgument.getAllMainSubTreeCon
|
||||
String cid = node.getUniqueName();
|
||||
|
||||
List<? extends INode> jobCatcherNodes = process.getNodesOfType("tJobStructureCatcher");
|
||||
boolean jobCatcherExists = jobCatcherNodes != null && !jobCatcherNodes.isEmpty();
|
||||
INode jobCatcherNode = jobCatcherExists ? jobCatcherNodes.get(0) : null;
|
||||
|
||||
boolean enableLogStash = !Boolean.getBoolean("deactivate_extended_component_log") && jobCatcherExists;
|
||||
boolean logstashCurrent = enableLogStash && !cid.startsWith("tJobStructureCatcher") && !cid.startsWith("talend");
|
||||
boolean enableLogStash = jobCatcherNodes != null && !jobCatcherNodes.isEmpty();
|
||||
boolean logstashCurrent = !cid.startsWith("tJobStructureCatcher") && !cid.startsWith("talend") && enableLogStash;
|
||||
|
||||
if((codePart.equals(ECodePart.END))&&(stat || logstashCurrent)){
|
||||
boolean iterateInVFComp = (node.getVirtualLinkTo() != null && node.getVirtualLinkTo() == EConnectionType.ITERATE);
|
||||
@@ -91,10 +88,10 @@ if((codePart.equals(ECodePart.END))&&(stat || logstashCurrent)){
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel.trim());
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel.trim());
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
@@ -125,10 +122,10 @@ if((codePart.equals(ECodePart.END))&&(stat || logstashCurrent)){
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel.trim());
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel.trim());
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
@@ -167,7 +164,6 @@ if((codePart.equals(ECodePart.END))&&(stat || logstashCurrent)){
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
if ((!node.isSubProcessStart())&&(NodeUtil.isDataAutoPropagated(node))) {
|
||||
if (inputColName!=null) {
|
||||
@@ -193,117 +189,6 @@ if((codePart.equals(ECodePart.END))&&(stat || logstashCurrent)){
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//log runtime lineage
|
||||
boolean enable_runtime_lineage_log = NodeUtil.isJobUsingRuntimeLineage(process) && jobCatcherExists && !cid.startsWith("tJobStructureCatcher") && !cid.startsWith("talend");
|
||||
if(enable_runtime_lineage_log) {//}
|
||||
|
||||
List<? extends IConnection> outConns = node.getOutgoingConnections();
|
||||
if(!outConns.isEmpty()) {
|
||||
%>
|
||||
if(tos_count_<%=node.getUniqueName() %> == 0) {
|
||||
<%
|
||||
//}
|
||||
}
|
||||
|
||||
for (IConnection conn : outConns) {
|
||||
if(!conn.getLineStyle().equals(EConnectionType.FLOW_MAIN) && !conn.getLineStyle().equals(EConnectionType.FLOW_MERGE) && !conn.getLineStyle().equals(EConnectionType.FLOW_REF)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
IMetadataTable metadata = conn.getMetadataTable();
|
||||
if (metadata==null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
List<IMetadataColumn> columns = metadata.getListColumns();
|
||||
|
||||
if(columns == null || columns.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
%>
|
||||
|
||||
class SchemaUtil_<%=conn.getUniqueName()%>_<%=conn.getMetadataTable().getTableName()%> {
|
||||
|
||||
public java.util.List<java.util.Map<String, String>> getSchema(final <%=NodeUtil.getPrivateConnClassName(conn) %>Struct <%=conn.getName()%>) {
|
||||
java.util.List<java.util.Map<String, String>> schema = new java.util.ArrayList<>();
|
||||
if(<%=conn.getName()%> == null) {
|
||||
return schema;
|
||||
}
|
||||
java.util.Map<String, String> field = null;
|
||||
<%
|
||||
for(IMetadataColumn column : columns){
|
||||
if("id_Dynamic".equals(column.getTalendType())) {
|
||||
%>
|
||||
routines.system.Dynamic dynamic = <%=conn.getName()%>.<%=column.getLabel()%>;
|
||||
if(dynamic != null) {
|
||||
for(routines.system.DynamicMetadata metadata : dynamic.metadatas) {
|
||||
field = new java.util.HashMap<>();
|
||||
field.put("name", metadata.getName());
|
||||
field.put("origin_name", metadata.getDbName());
|
||||
field.put("iskey", "" + metadata.isKey());
|
||||
field.put("talend_type", metadata.getType());
|
||||
field.put("type", metadata.getDbType());
|
||||
field.put("nullable", "" + metadata.isNullable());
|
||||
field.put("pattern", metadata.getFormat());
|
||||
field.put("length", "" + metadata.getLength());
|
||||
field.put("precision", "" + metadata.getPrecision());
|
||||
schema.add(field);
|
||||
}
|
||||
}
|
||||
|
||||
<%
|
||||
continue;
|
||||
}
|
||||
|
||||
String pattern = column.getPattern();
|
||||
if(pattern == null || pattern.isEmpty() || pattern.equals("\"\"")) {
|
||||
pattern = "\"\"";
|
||||
}
|
||||
%>
|
||||
field = new java.util.HashMap<>();
|
||||
field.put("name", "<%=column.getLabel()%>");
|
||||
field.put("origin_name", "<%=column.getOriginalDbColumnName()%>");
|
||||
field.put("iskey", "<%=column.isKey()%>");
|
||||
field.put("talend_type", "<%=column.getTalendType()%>");
|
||||
field.put("type", "<%=column.getType()%>");
|
||||
field.put("nullable", "<%=column.isNullable()%>");
|
||||
field.put("pattern", <%=pattern%>);
|
||||
field.put("length", "<%=column.getLength()%>");
|
||||
field.put("precision", "<%=column.getPrecision()%>");
|
||||
schema.add(field);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
return schema;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
java.util.List<java.util.Map<String, String>> schema_<%=conn.getUniqueName()%>_<%=conn.getMetadataTable().getTableName()%> = new SchemaUtil_<%=conn.getUniqueName()%>_<%=conn.getMetadataTable().getTableName()%>().getSchema(<%=conn.getName()%>);
|
||||
<%
|
||||
INode target = conn.getTarget();
|
||||
String targetNodeId = target.getUniqueName();
|
||||
String targetNodeComponent = target.getComponent().getName();
|
||||
|
||||
%>
|
||||
<%=jobCatcherNode.getUniqueName()%>.addConnectionSchemaMessage("<%=node.getUniqueName()%>","<%=node.getComponent().getName()%>",
|
||||
"<%=targetNodeId%>","<%=targetNodeComponent%>", "<%=conn.getUniqueName()%>" + iterateId, schema_<%=conn.getUniqueName()%>_<%=conn.getMetadataTable().getTableName()%>);
|
||||
<%=jobCatcherNode.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
<%
|
||||
}
|
||||
|
||||
if(!outConns.isEmpty()) {
|
||||
//{
|
||||
%>
|
||||
}
|
||||
<%
|
||||
}
|
||||
|
||||
//{
|
||||
}
|
||||
|
||||
//======================================TDI-17183 end=====================================
|
||||
boolean traceCodeGenerated = false;
|
||||
for (IConnection conn : node.getOutgoingConnections()) {
|
||||
|
||||
@@ -8,9 +8,6 @@
|
||||
org.talend.core.model.metadata.IMetadataColumn
|
||||
org.talend.core.model.process.EConnectionType
|
||||
org.talend.core.model.process.ElementParameterParser
|
||||
org.talend.core.model.process.EParameterFieldType
|
||||
org.talend.designer.core.model.components.EParameterName
|
||||
org.talend.designer.core.model.components.ElementParameter
|
||||
org.talend.designer.codegen.config.CodeGeneratorArgument
|
||||
org.talend.core.model.utils.NodeUtil
|
||||
org.talend.core.model.process.IConnectionCategory
|
||||
@@ -143,14 +140,10 @@
|
||||
connSet.addAll(node.getIncomingConnections(EConnectionType.FLOW_MAIN));
|
||||
connSet.addAll(node.getIncomingConnections(EConnectionType.FLOW_MERGE));
|
||||
|
||||
String cid = node.getUniqueName();
|
||||
|
||||
List<? extends INode> jobCatcherNodes = node.getProcess().getNodesOfType("tJobStructureCatcher");
|
||||
boolean jobCatcherExists = jobCatcherNodes != null && !jobCatcherNodes.isEmpty();
|
||||
INode jobCatcherNode = jobCatcherExists ? jobCatcherNodes.get(0) : null;
|
||||
|
||||
boolean enableLogStash = !Boolean.getBoolean("deactivate_extended_component_log") && jobCatcherExists;
|
||||
boolean logstashCurrent = enableLogStash && !cid.startsWith("tJobStructureCatcher") && !cid.startsWith("talend");
|
||||
boolean enableLogStash = jobCatcherNodes != null && !jobCatcherNodes.isEmpty();
|
||||
String cid = node.getUniqueName();
|
||||
boolean logstashCurrent = !cid.startsWith("tJobStructureCatcher") && !cid.startsWith("talend") && enableLogStash;
|
||||
|
||||
//about performance monitor, no way to support more than one job catcher component, also that is not necessary
|
||||
final String subprocessName4Catcher = logstashCurrent ? jobCatcherNodes.get(0).getDesignSubjobStartNode().getUniqueName() : null;
|
||||
@@ -197,10 +190,10 @@
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel.trim());
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel.trim());
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
%>
|
||||
@@ -233,10 +226,10 @@
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel.trim());
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel.trim());
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
%>
|
||||
@@ -260,10 +253,10 @@
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel.trim());
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel.trim());
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
|
||||
@@ -296,10 +289,10 @@
|
||||
String sourceNodeId = source.getUniqueName();
|
||||
|
||||
String sourceLabel = ElementParameterParser.getValue(source, "__LABEL__");
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel.trim());
|
||||
String sourceNodeLabel = ((sourceLabel==null || "__UNIQUE_NAME__".equals(sourceLabel) || sourceLabel.contains("\"")) ? sourceNodeId : sourceLabel);
|
||||
|
||||
String targetLabel = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel.trim());
|
||||
String targetNodeLabel = ((targetLabel==null || "__UNIQUE_NAME__".equals(targetLabel) || targetLabel.contains("\"")) ? node.getUniqueName() : targetLabel);
|
||||
|
||||
String sourceNodeComponent = source.getComponent().getName();
|
||||
|
||||
@@ -338,104 +331,11 @@
|
||||
<%
|
||||
log.startWork();
|
||||
log.logCompSetting();
|
||||
boolean enable_runtime_lineage_log = NodeUtil.isJobUsingRuntimeLineage(node.getProcess()) && jobCatcherExists && !cid.startsWith("tJobStructureCatcher") && !cid.startsWith("talend");
|
||||
if(enable_runtime_lineage_log) {
|
||||
%>
|
||||
class ParameterUtil_<%=cid%>{
|
||||
|
||||
public java.util.Map<String, String> getParameter() throws Exception{
|
||||
java.util.Map<String, String> component_parameters = new java.util.HashMap<>();
|
||||
<%
|
||||
java.util.Set<org.talend.core.model.process.EParameterFieldType> ignoredParamsTypes = new java.util.HashSet<org.talend.core.model.process.EParameterFieldType>();
|
||||
ignoredParamsTypes.addAll(
|
||||
java.util.Arrays.asList(
|
||||
org.talend.core.model.process.EParameterFieldType.SCHEMA_TYPE,
|
||||
org.talend.core.model.process.EParameterFieldType.SCHEMA_REFERENCE,
|
||||
org.talend.core.model.process.EParameterFieldType.LABEL,
|
||||
org.talend.core.model.process.EParameterFieldType.EXTERNAL,
|
||||
org.talend.core.model.process.EParameterFieldType.MAPPING_TYPE,
|
||||
org.talend.core.model.process.EParameterFieldType.IMAGE,
|
||||
org.talend.core.model.process.EParameterFieldType.TNS_EDITOR,
|
||||
org.talend.core.model.process.EParameterFieldType.WSDL2JAVA,
|
||||
org.talend.core.model.process.EParameterFieldType.GENERATEGRAMMARCONTROLLER,
|
||||
org.talend.core.model.process.EParameterFieldType.GENERATE_SURVIVORSHIP_RULES_CONTROLLER,
|
||||
org.talend.core.model.process.EParameterFieldType.REFRESH_REPORTS,
|
||||
org.talend.core.model.process.EParameterFieldType.BROWSE_REPORTS,
|
||||
org.talend.core.model.process.EParameterFieldType.PALO_DIM_SELECTION,
|
||||
org.talend.core.model.process.EParameterFieldType.GUESS_SCHEMA,
|
||||
org.talend.core.model.process.EParameterFieldType.MATCH_RULE_IMEX_CONTROLLER,
|
||||
org.talend.core.model.process.EParameterFieldType.MEMO_PERL,
|
||||
org.talend.core.model.process.EParameterFieldType.DBTYPE_LIST,
|
||||
org.talend.core.model.process.EParameterFieldType.VERSION,
|
||||
org.talend.core.model.process.EParameterFieldType.TECHNICAL,
|
||||
org.talend.core.model.process.EParameterFieldType.ICON_SELECTION,
|
||||
org.talend.core.model.process.EParameterFieldType.JAVA_COMMAND,
|
||||
org.talend.core.model.process.EParameterFieldType.TREE_TABLE,
|
||||
org.talend.core.model.process.EParameterFieldType.VALIDATION_RULE_TYPE,
|
||||
org.talend.core.model.process.EParameterFieldType.DCSCHEMA,
|
||||
org.talend.core.model.process.EParameterFieldType.SURVIVOR_RELATION,
|
||||
org.talend.core.model.process.EParameterFieldType.REST_RESPONSE_SCHEMA_TYPE,
|
||||
org.talend.core.model.process.EParameterFieldType.BUTTON
|
||||
)
|
||||
);
|
||||
for(org.talend.core.model.process.IElementParameter ep : org.talend.core.model.utils.NodeUtil.getDisplayedParameters(node)){
|
||||
if(!ep.isLog4JEnabled() || ignoredParamsTypes.contains(ep.getFieldType())){
|
||||
continue;
|
||||
}
|
||||
|
||||
ElementParameter p = (ElementParameter)ep;
|
||||
Object pluginValue = p.getTaggedValue("org.talend.sdk.component.source");
|
||||
if(pluginValue != null && String.class.cast(pluginValue).equalsIgnoreCase("tacokit")) {
|
||||
try {
|
||||
if (!(Boolean) org.talend.core.runtime.IAdditionalInfo.class.cast(p).func("isPersisted")) {
|
||||
continue;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
//do nothing
|
||||
}
|
||||
|
||||
%>
|
||||
<%@ include file="./tacokit_runtime_log.javajet"%>
|
||||
<%
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
String name = ep.getName();
|
||||
java.util.Set<String> ignoredParamsNames = new java.util.HashSet<String>();
|
||||
ignoredParamsNames.add("SQLPATTERN_VALUE");
|
||||
ignoredParamsNames.add("ADDITIONAL_INSERT_COLUMNS");
|
||||
ignoredParamsNames.add("ADDITIONAL_UPDATE_COLUMNS");
|
||||
ignoredParamsNames.add("SELECTION_TABLE");
|
||||
ignoredParamsNames.add("DIFFER_MESSAGE");
|
||||
ignoredParamsNames.add("NO_DIFFER_MESSAGE");
|
||||
|
||||
if(ignoredParamsNames.contains(name)) {
|
||||
//do nothing
|
||||
} else if(org.talend.core.model.process.EParameterFieldType.PASSWORD.equals(ep.getFieldType())){
|
||||
//not log password
|
||||
}else{
|
||||
String value = org.talend.core.model.utils.NodeUtil.getRuntimeParameterValue(node, ep);
|
||||
%>
|
||||
component_parameters.put("<%=name%>", String.valueOf(<%=value%>));
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
return component_parameters;
|
||||
}
|
||||
}
|
||||
|
||||
<%=jobCatcherNode.getUniqueName()%>.addComponentParameterMessage("<%=node.getUniqueName()%>", "<%=node.getComponent().getName()%>",
|
||||
new ParameterUtil_<%=cid%>().getParameter());
|
||||
<%=jobCatcherNode.getDesignSubjobStartNode().getUniqueName() %>Process(globalMap);
|
||||
<%
|
||||
}
|
||||
|
||||
if(logstashCurrent) {
|
||||
for (INode jobStructureCatcher : jobCatcherNodes) {
|
||||
String label = ElementParameterParser.getValue(node, "__LABEL__");
|
||||
String nodeLabel = ((label==null || "__UNIQUE_NAME__".equals(label) || label.contains("\"")) ? node.getUniqueName() : label.trim());
|
||||
String nodeLabel = ((label==null || "__UNIQUE_NAME__".equals(label) || label.contains("\"")) ? node.getUniqueName() : label);
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
<%=jobStructureCatcher.getUniqueName() %>.addCM("<%=node.getUniqueName()%>", "<%=nodeLabel%>", "<%=node.getComponent().getName()%>");
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
org.talend.core.model.process.ProcessUtils
|
||||
org.talend.core.model.components.IComponent
|
||||
org.talend.core.model.components.EComponentType
|
||||
org.talend.core.model.utils.NodeUtil
|
||||
"
|
||||
class="Footer"
|
||||
skeleton="footer_java.skeleton"
|
||||
@@ -519,63 +518,8 @@
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
boolean enableLogStash = !Boolean.getBoolean("deactivate_extended_component_log") && (jobCatcherNode!=null);
|
||||
boolean enable_runtime_lineage_log = NodeUtil.isJobUsingRuntimeLineage(process) && (jobCatcherNode!=null);
|
||||
if(enable_runtime_lineage_log) {
|
||||
%>
|
||||
java.util.Properties p_<%=jobCatcherNode.getUniqueName()%> = new java.util.Properties();
|
||||
p_<%=jobCatcherNode.getUniqueName()%>.setProperty("root.logger", "runtime");
|
||||
p_<%=jobCatcherNode.getUniqueName()%>.setProperty("encoding", "UTF-8");
|
||||
p_<%=jobCatcherNode.getUniqueName()%>.setProperty("application.name", "Talend Studio");
|
||||
p_<%=jobCatcherNode.getUniqueName()%>.setProperty("service.name", "Talend Studio Job");
|
||||
p_<%=jobCatcherNode.getUniqueName()%>.setProperty("instance.name", "Talend Studio Job Instance");
|
||||
p_<%=jobCatcherNode.getUniqueName()%>.setProperty("propagate.appender.exceptions", "none");
|
||||
p_<%=jobCatcherNode.getUniqueName()%>.setProperty("log.appender", "file");
|
||||
p_<%=jobCatcherNode.getUniqueName()%>.setProperty("appender.file.path", "runtime.json");
|
||||
p_<%=jobCatcherNode.getUniqueName()%>.setProperty("appender.file.maxsize", "52428800");
|
||||
p_<%=jobCatcherNode.getUniqueName()%>.setProperty("appender.file.maxbackup", "20");
|
||||
p_<%=jobCatcherNode.getUniqueName()%>.setProperty("host", "false");
|
||||
|
||||
final String runtime_dir_<%=jobCatcherNode.getUniqueName()%> = System.getProperty("runtime.lineage.outputpath");
|
||||
final String runtime_path_<%=jobCatcherNode.getUniqueName()%> = System.getProperty("runtime.lineage.appender.file.path");
|
||||
if(runtime_path_<%=jobCatcherNode.getUniqueName()%>==null || runtime_path_<%=jobCatcherNode.getUniqueName()%>.isEmpty()) {
|
||||
if(runtime_dir_<%=jobCatcherNode.getUniqueName()%>!=null && !runtime_dir_<%=jobCatcherNode.getUniqueName()%>.isEmpty()) {
|
||||
System.setProperty("runtime.lineage.appender.file.path",
|
||||
new StringBuilder().append(runtime_dir_<%=jobCatcherNode.getUniqueName()%>)
|
||||
.append((runtime_dir_<%=jobCatcherNode.getUniqueName()%>.endsWith("/") || runtime_dir_<%=jobCatcherNode.getUniqueName()%>.endsWith("\\")) ? "" : java.io.File.separator)
|
||||
.append(projectName)
|
||||
.append(java.io.File.separatorChar)
|
||||
.append(jobName)
|
||||
.append(java.io.File.separatorChar)
|
||||
.append(jobVersion)
|
||||
.append(java.io.File.separatorChar)
|
||||
.append("runtime_log_")
|
||||
.append(new java.text.SimpleDateFormat("yyyyMMddHHmmss").format(new java.util.Date()))
|
||||
.append(".json")
|
||||
.toString()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
System.getProperties().stringPropertyNames().stream()
|
||||
.filter(it -> it.startsWith("runtime.lineage.") && !"runtime.lineage.outputpath".equals(it))
|
||||
.forEach(key -> p_<%=jobCatcherNode.getUniqueName()%>.setProperty(key.substring("runtime.lineage.".length()), System.getProperty(key)));
|
||||
|
||||
<%if(isLog4j1Enabled) {%>
|
||||
org.apache.log4j.Logger.getLogger(p_<%=jobCatcherNode.getUniqueName()%>.getProperty("root.logger")).setLevel(org.apache.log4j.Level.DEBUG);
|
||||
<%}%>
|
||||
|
||||
<%if(isLog4j2Enabled) {%>
|
||||
org.apache.logging.log4j.core.config.Configurator.setLevel(p_<%=jobCatcherNode.getUniqueName()%>.getProperty("root.logger"), org.apache.logging.log4j.Level.DEBUG);
|
||||
<%}%>
|
||||
|
||||
runtime_lineage_logger_<%=jobCatcherNode.getUniqueName()%> = org.talend.job.audit.JobEventAuditLoggerFactory.createJobAuditLogger(p_<%=jobCatcherNode.getUniqueName()%>);
|
||||
<%
|
||||
}
|
||||
|
||||
if(enableLogStash) {
|
||||
String location = ElementParameterParser.getValue(jobCatcherNode, "__LOCATION__");
|
||||
if(jobCatcherNode!=null) {
|
||||
String location = ElementParameterParser.getValue(jobCatcherNode, "__LOCATION__");
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
java.util.Properties properties_<%=jobCatcherNode.getUniqueName()%> = new java.util.Properties();
|
||||
@@ -606,7 +550,7 @@
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
%>
|
||||
|
||||
if(clientHost == null) {
|
||||
clientHost = defaultClientHost;
|
||||
@@ -642,16 +586,6 @@
|
||||
}
|
||||
%>
|
||||
|
||||
boolean inOSGi = routines.system.BundleUtils.inOSGi();
|
||||
|
||||
if (inOSGi) {
|
||||
java.util.Dictionary<String, Object> jobProperties = routines.system.BundleUtils.getJobProperties(jobName);
|
||||
|
||||
if (jobProperties != null) {
|
||||
contextStr = (String)jobProperties.get("context");
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
//call job/subjob with an existing context, like: --context=production. if without this parameter, there will use the default context instead.
|
||||
java.io.InputStream inContext = <%=className%>.class.getClassLoader().getResourceAsStream("<%=jobClassPackageFolder%>/contexts/" + contextStr + ".properties");
|
||||
@@ -659,15 +593,13 @@
|
||||
inContext = <%=className%>.class.getClassLoader().getResourceAsStream("config/contexts/" + contextStr + ".properties");
|
||||
}
|
||||
if (inContext != null) {
|
||||
try {
|
||||
//defaultProps is in order to keep the original context value
|
||||
if(context != null && context.isEmpty()) {
|
||||
//defaultProps is in order to keep the original context value
|
||||
if(context != null && context.isEmpty()) {
|
||||
defaultProps.load(inContext);
|
||||
context = new ContextProperties(defaultProps);
|
||||
}
|
||||
} finally {
|
||||
inContext.close();
|
||||
}
|
||||
|
||||
inContext.close();
|
||||
} else if (!isDefaultContext) {
|
||||
//print info and job continue to run, for case: context_param is not empty.
|
||||
System.err.println("Could not find the context " + contextStr);
|
||||
@@ -1014,7 +946,7 @@ this.globalResumeTicket = true;//to run tPreJob
|
||||
<%
|
||||
}
|
||||
|
||||
if(enableLogStash) {
|
||||
if(jobCatcherNode!=null) {
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
<%=jobCatcherNode.getUniqueName() %>.addJobStartMessage();
|
||||
@@ -1167,7 +1099,7 @@ this.globalResumeTicket = true;//to run tPostJob
|
||||
<%
|
||||
}
|
||||
|
||||
if(enableLogStash) {
|
||||
if(jobCatcherNode!=null) {
|
||||
%>
|
||||
if(enableLogStash) {
|
||||
<%=jobCatcherNode.getUniqueName() %>.addJobEndMessage(startTime, end, status);
|
||||
@@ -1206,24 +1138,6 @@ if (execStat) {
|
||||
}
|
||||
%>
|
||||
int returnCode = 0;
|
||||
|
||||
<%
|
||||
if (isRunInMultiThread) {
|
||||
%>
|
||||
Integer localErrorCode = (Integer)(((java.util.Map)threadLocal.get()).get("errorCode"));
|
||||
String localStatus = (String)(((java.util.Map)threadLocal.get()).get("status"));
|
||||
if (localErrorCode != null) {
|
||||
if (errorCode == null || localErrorCode.compareTo(errorCode) > 0) {
|
||||
errorCode = localErrorCode;
|
||||
}
|
||||
}
|
||||
if (localStatus != null && !status.equals("failure")){
|
||||
status = localStatus;
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
if(errorCode == null) {
|
||||
returnCode = status != null && status.equals("failure") ? 1 : 0;
|
||||
} else {
|
||||
|
||||
@@ -60,10 +60,6 @@ if ((metadatas != null) && (metadatas.size() > 0)) { // metadata
|
||||
|
||||
org.talend.components.api.component.ComponentDefinition def_<%=cid %> =
|
||||
new <%= def.getClass().getName()%>();
|
||||
|
||||
org.talend.components.api.component.runtime.Writer writer_<%=cid%> = null;
|
||||
org.talend.components.api.component.runtime.Reader reader_<%=cid%> = null;
|
||||
|
||||
<%
|
||||
List<Component.CodegenPropInfo> propsToProcess = component.getCodegenPropInfos(componentProps);
|
||||
%>
|
||||
@@ -244,11 +240,11 @@ if(isTopologyNone) {
|
||||
|
||||
if (hasOutputOnly || asInputComponent) {
|
||||
%>
|
||||
if (sourceOrSink_<%=cid%> instanceof org.talend.components.api.component.runtime.Source) {
|
||||
org.talend.components.api.component.runtime.Source source_<%=cid%> =
|
||||
(org.talend.components.api.component.runtime.Source)sourceOrSink_<%=cid%>;
|
||||
reader_<%=cid%> = source_<%=cid%>.createReader(container_<%=cid%>);
|
||||
reader_<%=cid%> = new org.talend.codegen.flowvariables.runtime.FlowVariablesReader(reader_<%=cid%>, container_<%=cid%>);
|
||||
org.talend.components.api.component.runtime.Source source_<%=cid%> =
|
||||
(org.talend.components.api.component.runtime.Source)sourceOrSink_<%=cid%>;
|
||||
org.talend.components.api.component.runtime.Reader reader_<%=cid%> =
|
||||
source_<%=cid%>.createReader(container_<%=cid%>);
|
||||
reader_<%=cid%> = new org.talend.codegen.flowvariables.runtime.FlowVariablesReader(reader_<%=cid%>, container_<%=cid%>);
|
||||
|
||||
<%
|
||||
IConnection main = null;
|
||||
@@ -270,19 +266,19 @@ if (hasOutputOnly || asInputComponent) {
|
||||
IConnection schemaSourceConnector = (main!=null ? main : reject);
|
||||
String schemaSourceConnectorName = schemaSourceConnector.getMetadataTable().getAttachedConnector();
|
||||
%>
|
||||
boolean multi_output_is_allowed_<%=cid%> = false;
|
||||
boolean multi_output_is_allowed_<%=cid%> = false;
|
||||
<% //take care SourceOrSink.validate will change the schema if it contains include-all-fields, so need to get design Avro schema before validate %>
|
||||
org.talend.components.api.component.Connector c_<%=cid%> = null;
|
||||
for (org.talend.components.api.component.Connector currentConnector : props_<%=cid %>.getAvailableConnectors(null, true)) {
|
||||
if (currentConnector.getName().equals("<%=schemaSourceConnectorName%>")) {
|
||||
c_<%=cid%> = currentConnector;
|
||||
}
|
||||
|
||||
if (currentConnector.getName().equals("REJECT")) {//it's better to move the code to javajet
|
||||
multi_output_is_allowed_<%=cid%> = true;
|
||||
}
|
||||
org.talend.components.api.component.Connector c_<%=cid%> = null;
|
||||
for (org.talend.components.api.component.Connector currentConnector : props_<%=cid %>.getAvailableConnectors(null, true)) {
|
||||
if (currentConnector.getName().equals("<%=schemaSourceConnectorName%>")) {
|
||||
c_<%=cid%> = currentConnector;
|
||||
}
|
||||
org.apache.avro.Schema schema_<%=cid%> = props_<%=cid %>.getSchema(c_<%=cid%>, true);
|
||||
|
||||
if (currentConnector.getName().equals("REJECT")) {//it's better to move the code to javajet
|
||||
multi_output_is_allowed_<%=cid%> = true;
|
||||
}
|
||||
}
|
||||
org.apache.avro.Schema schema_<%=cid%> = props_<%=cid %>.getSchema(c_<%=cid%>, true);
|
||||
|
||||
<%
|
||||
irToRow = new IndexedRecordToRowStructGenerator(cid, null, columnList);
|
||||
@@ -290,119 +286,117 @@ if (hasOutputOnly || asInputComponent) {
|
||||
}
|
||||
%>
|
||||
|
||||
// Iterate through the incoming data.
|
||||
boolean available_<%=cid%> = reader_<%=cid%>.start();
|
||||
// Iterate through the incoming data.
|
||||
boolean available_<%=cid%> = reader_<%=cid%>.start();
|
||||
|
||||
resourceMap.put("reader_<%=cid%>", reader_<%=cid%>);
|
||||
resourceMap.put("reader_<%=cid%>", reader_<%=cid%>);
|
||||
|
||||
for (; available_<%=cid%>; available_<%=cid%> = reader_<%=cid%>.advance()) {
|
||||
nb_line_<%=cid %>++;
|
||||
|
||||
<%if(hasDataOutput) {%>
|
||||
if (multi_output_is_allowed_<%=cid%>) {
|
||||
<%if(main!=null){%>
|
||||
<%=main.getName()%> = null;
|
||||
<%}%>
|
||||
|
||||
<%if(reject!=null){%>
|
||||
<%=reject.getName()%> = null;
|
||||
<%}%>
|
||||
}
|
||||
<%}%>
|
||||
for (; available_<%=cid%>; available_<%=cid%> = reader_<%=cid%>.advance()) {
|
||||
nb_line_<%=cid %>++;
|
||||
|
||||
try {
|
||||
Object data_<%=cid%> = reader_<%=cid%>.getCurrent();
|
||||
<%
|
||||
if (main != null) {
|
||||
%>
|
||||
|
||||
if(multi_output_is_allowed_<%=cid%>) {
|
||||
<%=main.getName()%> = new <%=main.getName() %>Struct();
|
||||
}
|
||||
|
||||
<%
|
||||
irToRow.generateConvertRecord("data_" + cid, main.getName(), main.getMetadataTable().getListColumns());
|
||||
}
|
||||
%>
|
||||
} catch (org.talend.components.api.exception.DataRejectException e_<%=cid%>) {
|
||||
java.util.Map<String,Object> info_<%=cid%> = e_<%=cid%>.getRejectInfo();
|
||||
<%
|
||||
if (reject!=null) {
|
||||
%>
|
||||
Object data_<%=cid%> = info_<%=cid%>.get("talend_record");
|
||||
|
||||
if (multi_output_is_allowed_<%=cid%>) {
|
||||
<%=reject.getName()%> = new <%=reject.getName() %>Struct();
|
||||
}
|
||||
try{
|
||||
<%
|
||||
irToRow.generateConvertRecord("data_" + cid, reject.getName());
|
||||
%>
|
||||
}catch(java.lang.Exception e){
|
||||
// do nothing
|
||||
}
|
||||
<%
|
||||
Set<String> commonColumns = new HashSet<String>();
|
||||
|
||||
for (IMetadataColumn column : columnList) {
|
||||
commonColumns.add(column.getLabel());
|
||||
}
|
||||
|
||||
//pass error columns
|
||||
List<IMetadataColumn> rejectColumns = reject.getMetadataTable().getListColumns();
|
||||
for(IMetadataColumn column : rejectColumns) {
|
||||
String columnName = column.getLabel();
|
||||
|
||||
// JavaType javaType = JavaTypesManager.getJavaTypeFromId(column.getTalendType());
|
||||
String typeToGenerate = JavaTypesManager.getTypeToGenerate(column.getTalendType(), column.isNullable());
|
||||
|
||||
//error columns
|
||||
if(!commonColumns.contains(columnName)) {
|
||||
%>
|
||||
<%=reject.getName()%>.<%=columnName%> = (<%=typeToGenerate%>)info_<%=cid%>.get("<%=columnName%>");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
//TODO use a method instead of getting method by the special key "error/errorMessage"
|
||||
Object errorMessage_<%=cid%> = null;
|
||||
if(info_<%=cid%>.containsKey("error")){
|
||||
errorMessage_<%=cid%> = info_<%=cid%>.get("error");
|
||||
}else if(info_<%=cid%>.containsKey("errorMessage")){
|
||||
errorMessage_<%=cid%> = info_<%=cid%>.get("errorMessage");
|
||||
}else{
|
||||
errorMessage_<%=cid%> = "Rejected but error message missing";
|
||||
}
|
||||
errorMessage_<%=cid%> = "Row "+ nb_line_<%=cid %> + ": "+errorMessage_<%=cid%>;
|
||||
System.err.println(errorMessage_<%=cid%>);
|
||||
<%
|
||||
}
|
||||
if (main != null) {
|
||||
%>
|
||||
// If the record is reject, the main line record should put NULL
|
||||
<%=main.getName()%> = null;
|
||||
<%
|
||||
}
|
||||
%>
|
||||
} // end of catch
|
||||
<%if(hasDataOutput) {%>
|
||||
if (multi_output_is_allowed_<%=cid%>) {
|
||||
<%if(main!=null){%>
|
||||
<%=main.getName()%> = null;
|
||||
<%}%>
|
||||
|
||||
<%if(reject!=null){%>
|
||||
<%=reject.getName()%> = null;
|
||||
<%}%>
|
||||
}
|
||||
<%}%>
|
||||
|
||||
try {
|
||||
Object data_<%=cid%> = reader_<%=cid%>.getCurrent();
|
||||
<%
|
||||
if (main != null) {
|
||||
%>
|
||||
|
||||
if(multi_output_is_allowed_<%=cid%>) {
|
||||
<%=main.getName()%> = new <%=main.getName() %>Struct();
|
||||
}
|
||||
|
||||
<%
|
||||
irToRow.generateConvertRecord("data_" + cid, main.getName(), main.getMetadataTable().getListColumns());
|
||||
}
|
||||
%>
|
||||
} catch (org.talend.components.api.exception.DataRejectException e_<%=cid%>) {
|
||||
java.util.Map<String,Object> info_<%=cid%> = e_<%=cid%>.getRejectInfo();
|
||||
<%
|
||||
if (reject!=null) {
|
||||
%>
|
||||
Object data_<%=cid%> = info_<%=cid%>.get("talend_record");
|
||||
|
||||
if (multi_output_is_allowed_<%=cid%>) {
|
||||
<%=reject.getName()%> = new <%=reject.getName() %>Struct();
|
||||
}
|
||||
try{
|
||||
<%
|
||||
irToRow.generateConvertRecord("data_" + cid, reject.getName());
|
||||
%>
|
||||
}catch(java.lang.Exception e){
|
||||
// do nothing
|
||||
}
|
||||
<%
|
||||
Set<String> commonColumns = new HashSet<String>();
|
||||
|
||||
for (IMetadataColumn column : columnList) {
|
||||
commonColumns.add(column.getLabel());
|
||||
}
|
||||
|
||||
//pass error columns
|
||||
List<IMetadataColumn> rejectColumns = reject.getMetadataTable().getListColumns();
|
||||
for(IMetadataColumn column : rejectColumns) {
|
||||
String columnName = column.getLabel();
|
||||
|
||||
// JavaType javaType = JavaTypesManager.getJavaTypeFromId(column.getTalendType());
|
||||
String typeToGenerate = JavaTypesManager.getTypeToGenerate(column.getTalendType(), column.isNullable());
|
||||
|
||||
//error columns
|
||||
if(!commonColumns.contains(columnName)) {
|
||||
%>
|
||||
<%=reject.getName()%>.<%=columnName%> = (<%=typeToGenerate%>)info_<%=cid%>.get("<%=columnName%>");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
%>
|
||||
//TODO use a method instead of getting method by the special key "error/errorMessage"
|
||||
Object errorMessage_<%=cid%> = null;
|
||||
if(info_<%=cid%>.containsKey("error")){
|
||||
errorMessage_<%=cid%> = info_<%=cid%>.get("error");
|
||||
}else if(info_<%=cid%>.containsKey("errorMessage")){
|
||||
errorMessage_<%=cid%> = info_<%=cid%>.get("errorMessage");
|
||||
}else{
|
||||
errorMessage_<%=cid%> = "Rejected but error message missing";
|
||||
}
|
||||
errorMessage_<%=cid%> = "Row "+ nb_line_<%=cid %> + ": "+errorMessage_<%=cid%>;
|
||||
System.err.println(errorMessage_<%=cid%>);
|
||||
<%
|
||||
}
|
||||
if (main != null) {
|
||||
%>
|
||||
// If the record is reject, the main line record should put NULL
|
||||
<%=main.getName()%> = null;
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
<%
|
||||
// The for loop around the incoming records from the reader is left open.
|
||||
|
||||
|
||||
} else if (hasInput) {
|
||||
%>
|
||||
org.talend.codegen.enforcer.IncomingSchemaEnforcer incomingEnforcer_<%=cid%> = null;
|
||||
if (sourceOrSink_<%=cid%> instanceof org.talend.components.api.component.runtime.Sink) {
|
||||
org.talend.components.api.component.runtime.Sink sink_<%=cid%> =
|
||||
(org.talend.components.api.component.runtime.Sink)sourceOrSink_<%=cid%>;
|
||||
org.talend.components.api.component.runtime.WriteOperation writeOperation_<%=cid%> = sink_<%=cid%>.createWriteOperation();
|
||||
writeOperation_<%=cid%>.initialize(container_<%=cid%>);
|
||||
writer_<%=cid%> = writeOperation_<%=cid%>.createWriter(container_<%=cid%>);
|
||||
writer_<%=cid%>.open("<%=cid%>");
|
||||
org.talend.components.api.component.runtime.Sink sink_<%=cid%> =
|
||||
(org.talend.components.api.component.runtime.Sink)sourceOrSink_<%=cid%>;
|
||||
org.talend.components.api.component.runtime.WriteOperation writeOperation_<%=cid%> = sink_<%=cid%>.createWriteOperation();
|
||||
writeOperation_<%=cid%>.initialize(container_<%=cid%>);
|
||||
org.talend.components.api.component.runtime.Writer writer_<%=cid%> = writeOperation_<%=cid%>.createWriter(container_<%=cid%>);
|
||||
writer_<%=cid%>.open("<%=cid%>");
|
||||
|
||||
resourceMap.put("writer_<%=cid%>", writer_<%=cid%>);
|
||||
|
||||
resourceMap.put("writer_<%=cid%>", writer_<%=cid%>);
|
||||
} // end of "sourceOrSink_<%=cid%> instanceof ...Sink"
|
||||
org.talend.components.api.component.Connector c_<%=cid%> = null;
|
||||
for (org.talend.components.api.component.Connector currentConnector : props_<%=cid %>.getAvailableConnectors(null, false)) {
|
||||
if (currentConnector.getName().equals("MAIN")) {
|
||||
@@ -411,7 +405,8 @@ if (hasOutputOnly || asInputComponent) {
|
||||
}
|
||||
}
|
||||
org.apache.avro.Schema designSchema_<%=cid%> = props_<%=cid %>.getSchema(c_<%=cid%>, false);
|
||||
incomingEnforcer_<%=cid%> = new org.talend.codegen.enforcer.IncomingSchemaEnforcer(designSchema_<%=cid%>);
|
||||
org.talend.codegen.enforcer.IncomingSchemaEnforcer incomingEnforcer_<%=cid%>
|
||||
= new org.talend.codegen.enforcer.IncomingSchemaEnforcer(designSchema_<%=cid%>);
|
||||
<%
|
||||
List<? extends IConnection> outgoingConns = node.getOutgoingSortedConnections();
|
||||
if (outgoingConns!=null){
|
||||
@@ -447,8 +442,7 @@ if (hasOutputOnly || asInputComponent) {
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
%>
|
||||
java.lang.Iterable<?> outgoingMainRecordsList_<%=cid%> = new java.util.ArrayList<Object>();
|
||||
java.util.Iterator outgoingMainRecordsIt_<%=cid%> = null;
|
||||
|
||||
|
||||
@@ -58,24 +58,13 @@ if(isTopologyNone) {
|
||||
else if(hasOutputOnly || asInputComponent){
|
||||
%>
|
||||
} // while
|
||||
<%
|
||||
if (hasOutputOnly || asInputComponent) {
|
||||
%>
|
||||
} // end of "if (sourceOrSink_<%=cid%> instanceof ...Source)"
|
||||
<% } %>
|
||||
java.util.Map<String, Object> resultMap_<%=cid%> = null;
|
||||
if (reader_<%=cid%> != null) {
|
||||
reader_<%=cid%>.close();
|
||||
resultMap_<%=cid%> = reader_<%=cid%>.getReturnValues();
|
||||
}
|
||||
reader_<%=cid%>.close();
|
||||
final java.util.Map<String, Object> resultMap_<%=cid%> = reader_<%=cid%>.getReturnValues();
|
||||
<%
|
||||
}else if(hasInput){
|
||||
%>
|
||||
java.util.Map<String, Object> resultMap_<%=cid%> = null;
|
||||
if (writer_<%=cid%> != null) {
|
||||
org.talend.components.api.component.runtime.Result resultObject_<%=cid%> = (org.talend.components.api.component.runtime.Result)writer_<%=cid%>.close();
|
||||
resultMap_<%=cid%> = writer_<%=cid%>.getWriteOperation().finalize(java.util.Arrays.<org.talend.components.api.component.runtime.Result>asList(resultObject_<%=cid%>), container_<%=cid%>);
|
||||
}
|
||||
org.talend.components.api.component.runtime.Result resultObject_<%=cid%> = (org.talend.components.api.component.runtime.Result)writer_<%=cid%>.close();
|
||||
final java.util.Map<String, Object> resultMap_<%=cid%> = writer_<%=cid%>.getWriteOperation().finalize(java.util.Arrays.<org.talend.components.api.component.runtime.Result>asList(resultObject_<%=cid%>), container_<%=cid%>);
|
||||
<%
|
||||
} else {
|
||||
return stringBuffer.toString();
|
||||
|
||||
@@ -84,7 +84,7 @@ if(hasInput){
|
||||
for (int i = 0; i < input_columnList.size(); i++) {
|
||||
if(!input_columnList.get(i).getTalendType().equals("id_Dynamic")) {
|
||||
%>
|
||||
if (incomingEnforcer_<%=cid%> != null && incomingEnforcer_<%=cid%>.getDesignSchema().getField("<%=input_columnList.get(i)%>") == null){
|
||||
if (incomingEnforcer_<%=cid%>.getDesignSchema().getField("<%=input_columnList.get(i)%>") == null){
|
||||
incomingEnforcer_<%=cid%>.addIncomingNodeField("<%=input_columnList.get(i)%>", ((Object) <%=inputConn.getName()%>.<%=input_columnList.get(i)%>).getClass().getCanonicalName());
|
||||
shouldCreateRuntimeSchemaForIncomingNode = true;
|
||||
}
|
||||
@@ -92,7 +92,7 @@ if(hasInput){
|
||||
}
|
||||
}
|
||||
%>
|
||||
if (shouldCreateRuntimeSchemaForIncomingNode && incomingEnforcer_<%=cid%> != null){
|
||||
if (shouldCreateRuntimeSchemaForIncomingNode){
|
||||
incomingEnforcer_<%=cid%>.createRuntimeSchema();
|
||||
}
|
||||
<%
|
||||
@@ -111,7 +111,7 @@ if(hasInput){
|
||||
|
||||
if (dynamicPos != -1) {
|
||||
%>
|
||||
if (incomingEnforcer_<%=cid%> != null && !incomingEnforcer_<%=cid%>.areDynamicFieldsInitialized()) {
|
||||
if (!incomingEnforcer_<%=cid%>.areDynamicFieldsInitialized()) {
|
||||
// Initialize the dynamic columns when they are first encountered.
|
||||
for (routines.system.DynamicMetadata dm_<%=cid%> : <%=inputConn.getName()%>.<%=input_columnList.get(dynamicPos).getLabel()%>.metadatas) {
|
||||
incomingEnforcer_<%=cid%>.addDynamicField(
|
||||
@@ -128,26 +128,22 @@ if(hasInput){
|
||||
}
|
||||
|
||||
%>
|
||||
if (incomingEnforcer_<%=cid%> != null) {
|
||||
incomingEnforcer_<%=cid%>.createNewRecord();
|
||||
}
|
||||
incomingEnforcer_<%=cid%>.createNewRecord();
|
||||
<%
|
||||
for (int i = 0; i < input_columnList.size(); i++) { // column
|
||||
IMetadataColumn column = input_columnList.get(i);
|
||||
if (dynamicPos != i) {
|
||||
%>
|
||||
//skip the put action if the input column doesn't appear in component runtime schema
|
||||
if (incomingEnforcer_<%=cid%> != null && incomingEnforcer_<%=cid%>.getRuntimeSchema().getField("<%=input_columnList.get(i)%>") != null){
|
||||
if (incomingEnforcer_<%=cid%>.getRuntimeSchema().getField("<%=input_columnList.get(i)%>") != null){
|
||||
incomingEnforcer_<%=cid%>.put("<%=column.getLabel()%>", <%=inputConn.getName()%>.<%=column.getLabel()%>);
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
if (incomingEnforcer_<%=cid%> != null) {
|
||||
for (int i = 0; i < <%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnCount(); i++) {
|
||||
incomingEnforcer_<%=cid%>.put(<%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnMetadata(i).getName(),
|
||||
<%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnValue(i));
|
||||
}
|
||||
for (int i = 0; i < <%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnCount(); i++) {
|
||||
incomingEnforcer_<%=cid%>.put(<%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnMetadata(i).getName(),
|
||||
<%=inputConn.getName()%>.<%=column.getLabel()%>.getColumnValue(i));
|
||||
}
|
||||
<%
|
||||
}
|
||||
@@ -181,11 +177,7 @@ if(hasInput){
|
||||
} // propInfo
|
||||
|
||||
%>
|
||||
|
||||
org.apache.avro.generic.IndexedRecord data_<%=cid%> = null;
|
||||
if (incomingEnforcer_<%=cid%> != null) {
|
||||
data_<%=cid%> = incomingEnforcer_<%=cid%>.getCurrentRecord();
|
||||
}
|
||||
org.apache.avro.generic.IndexedRecord data_<%=cid%> = incomingEnforcer_<%=cid%>.getCurrentRecord();
|
||||
|
||||
<%
|
||||
boolean isParallelize ="true".equalsIgnoreCase(ElementParameterParser.getValue(node, "__PARALLELIZE__"));
|
||||
@@ -198,9 +190,8 @@ if(hasInput){
|
||||
}
|
||||
}
|
||||
%>
|
||||
if (writer_<%=cid%> != null && data_<%=cid%> != null) {
|
||||
writer_<%=cid%>.write(data_<%=cid%>);
|
||||
}
|
||||
|
||||
writer_<%=cid%>.write(data_<%=cid%>);
|
||||
|
||||
nb_line_<%=cid %>++;
|
||||
<%if(hasMainOutput){
|
||||
|
||||
@@ -389,26 +389,13 @@ public <%=JavaTypesManager.getTypeToGenerate(ctxParam.getType(),true)%> get<%=Ch
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
boolean enableLogStash = !Boolean.getBoolean("deactivate_extended_component_log") && (jobCatcherNode!=null);
|
||||
boolean enable_runtime_lineage_log = NodeUtil.isJobUsingRuntimeLineage(process) && (jobCatcherNode!=null);
|
||||
|
||||
if(jobCatcherNode!=null) {
|
||||
%>
|
||||
private final JobStructureCatcherUtils <%=jobCatcherNode.getUniqueName() %> = new JobStructureCatcherUtils(jobName, "<%=process.getId() %>", "<%=process.getVersion() %>");
|
||||
<%
|
||||
}
|
||||
if(enable_runtime_lineage_log) {
|
||||
%>
|
||||
private org.talend.job.audit.JobAuditLogger runtime_lineage_logger_<%=jobCatcherNode.getUniqueName()%> = null;
|
||||
<%
|
||||
}
|
||||
|
||||
boolean enableLogStash = jobCatcherNode != null;
|
||||
if (enableLogStash) {
|
||||
%>
|
||||
private final JobStructureCatcherUtils <%=jobCatcherNode.getUniqueName() %> = new JobStructureCatcherUtils(jobName, "<%=process.getId() %>", "<%=process.getVersion() %>");
|
||||
private org.talend.job.audit.JobAuditLogger auditLogger_<%=jobCatcherNode.getUniqueName()%> = null;
|
||||
|
||||
private RunStat runStat = new RunStat(<%=jobCatcherNode.getUniqueName() %>, System.getProperty("audit.interval"));
|
||||
private RunStat runStat = new RunStat(<%=jobCatcherNode.getUniqueName() %>);
|
||||
<%
|
||||
} else if(stats) {
|
||||
%>
|
||||
@@ -437,20 +424,6 @@ private RunTrace runTrace = new RunTrace();
|
||||
globalMap.put(KEY_DB_DATASOURCES, talendDataSources);
|
||||
globalMap.put(KEY_DB_DATASOURCES_RAW, new java.util.HashMap<String, javax.sql.DataSource>(dataSources));
|
||||
}
|
||||
|
||||
public void setDataSourceReferences(List serviceReferences) throws Exception{
|
||||
|
||||
java.util.Map<String, routines.system.TalendDataSource> talendDataSources = new java.util.HashMap<String, routines.system.TalendDataSource>();
|
||||
java.util.Map<String, javax.sql.DataSource> dataSources = new java.util.HashMap<String, javax.sql.DataSource>();
|
||||
|
||||
for (java.util.Map.Entry<String, javax.sql.DataSource> entry : BundleUtils.getServices(serviceReferences, javax.sql.DataSource.class).entrySet()) {
|
||||
dataSources.put(entry.getKey(), entry.getValue());
|
||||
talendDataSources.put(entry.getKey(), new routines.system.TalendDataSource(entry.getValue()));
|
||||
}
|
||||
|
||||
globalMap.put(KEY_DB_DATASOURCES, talendDataSources);
|
||||
globalMap.put(KEY_DB_DATASOURCES_RAW, new java.util.HashMap<String, javax.sql.DataSource>(dataSources));
|
||||
}
|
||||
|
||||
<%
|
||||
for (INode logCatcher : process.getNodesOfType("tLogCatcher")) {
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
org.talend.designer.codegen.config.CodeGeneratorArgument
|
||||
org.talend.designer.codegen.config.NodesSubTree
|
||||
org.talend.core.model.process.IProcess
|
||||
org.talend.core.model.process.ProcessUtils
|
||||
org.talend.core.model.utils.NodeUtil
|
||||
org.talend.core.model.process.IContextParameter
|
||||
java.util.List
|
||||
@@ -29,13 +28,7 @@ INode node = (INode)codeGenArgument.getArgument();
|
||||
boolean containsTPartitioner = node.getProcess().getNodesOfType("tPartitioner").size() > 0 ? true : false;
|
||||
boolean isRunJob = "tRunJob".equals(node.getComponent().getName());
|
||||
IProcess process = node.getProcess();
|
||||
|
||||
boolean isTestContainer=ProcessUtils.isTestContainer(process);
|
||||
String className = process.getName();
|
||||
if (isTestContainer) {
|
||||
className = className + "Test";
|
||||
}
|
||||
|
||||
|
||||
NodesSubTree subTree = (NodesSubTree) codeGenArgument.getSubTree();
|
||||
ECodePart codePart = codeGenArgument.getCodePart();
|
||||
//boolean trace = codeGenArgument.isTrace();
|
||||
@@ -85,10 +78,10 @@ for (IConnection iterateConn : iterateConnSet) { //1
|
||||
@Override
|
||||
public Object put(String key, Object value) {
|
||||
<%if(!isRunInMultiThread){%>
|
||||
synchronized (<%=className%>.this.obj) {
|
||||
synchronized (<%=process.getName()%>.this.obj) {
|
||||
<%}%>
|
||||
super.put(key, value);
|
||||
return <%=className%>.this.globalMap.put(key, value);
|
||||
return <%=process.getName()%>.this.globalMap.put(key, value);
|
||||
<%if(!isRunInMultiThread){%>
|
||||
}
|
||||
<%}%>
|
||||
@@ -165,7 +158,7 @@ for (IConnection iterateConn : iterateConnSet) { //1
|
||||
synchronized (globalMap) {
|
||||
this.globalMap = java.util.Collections.synchronizedMap(new ThreadedMap(globalMap));
|
||||
<%}else{%>
|
||||
synchronized (<%=className%>.this.obj) {
|
||||
synchronized (<%=process.getName()%>.this.obj) {
|
||||
this.globalMap = new ThreadedMap(globalMap);
|
||||
<%}%>
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@
|
||||
INode startNode = subTree.getRootNode();
|
||||
String startNodeId = startNode.getUniqueName();
|
||||
|
||||
if ("tCollector".equals( startNode.getComponent().getOriginalName() )) {
|
||||
if(startNodeId!=null && startNodeId.startsWith("tCollector")) {
|
||||
List<? extends INode> departitioners = startNode.getProcess().getNodesOfType("tDepartitioner");
|
||||
if(departitioners!=null) {
|
||||
for(INode departitioner : departitioners) {
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
<%
|
||||
//copy from configuration.javajet for tacokit
|
||||
%>
|
||||
|
||||
<%
|
||||
//TODO: modify this part for Maps and nested lists.
|
||||
if (p.getFieldType() == EParameterFieldType.TABLE || p.getFieldType() == EParameterFieldType.TACOKIT_SUGGESTABLE_TABLE) {
|
||||
java.util.List<java.util.Map<String, String>> tableValues = ElementParameterParser.createTableValues((java.util.List<java.util.Map<String, Object>>) p.getValue(), p);
|
||||
String[] items = p.getListItemsDisplayCodeName();
|
||||
String tableName = p.getName().replace('$', '.');
|
||||
boolean primitiveTable = items.length == 1 && items[0].equals(tableName + "[]");
|
||||
String tableNamePrefix = tableName + "[]";
|
||||
for (int i = 0; i < tableValues.size(); i++) {
|
||||
java.util.Map<String, String> lineValues = tableValues.get(i);
|
||||
for (int j = 0; j < items.length; j++) {
|
||||
String key = tableName + "[" + i + "]";
|
||||
if (!primitiveTable) {
|
||||
final String columnName = items[j].substring(tableNamePrefix.length(), items[j].length());
|
||||
key = key + columnName;
|
||||
}
|
||||
String value = lineValues.get(items[j]);
|
||||
if (!org.talend.core.model.utils.ContextParameterUtils.isDynamic(value)) {
|
||||
value = org.talend.core.model.utils.TalendTextUtils.removeQuotes(value);
|
||||
value = org.talend.core.model.utils.TalendTextUtils.addQuotes(value);
|
||||
}
|
||||
|
||||
if(value==null || "null".equals(value.trim())) {
|
||||
value = "(Object)null";
|
||||
}
|
||||
%>
|
||||
component_parameters.put("<%=key%>",String.valueOf(<%=value%>));
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else if(p.getFieldType() == EParameterFieldType.SCHEMA_TYPE) {
|
||||
final String parameterName = p.getName();
|
||||
IConnection connection = null;
|
||||
final List<? extends IConnection> connections = NodeUtil.getOutgoingConnections(node, p.getContext());
|
||||
if(connections != null && !connections.isEmpty()) {
|
||||
connection = connections.get(0);
|
||||
}
|
||||
if(connection != null) {
|
||||
IMetadataTable metaTable = connection.getMetadataTable();
|
||||
List<IMetadataColumn> columns = metaTable.getListColumns();
|
||||
for(int i = 0; i < columns.size(); i++) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
%>
|
||||
component_parameters.put("<%=parameterName%>[<%=i%>]", "<%=column.getLabel()%>");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else if (p.getFieldType() == EParameterFieldType.TACOKIT_INPUT_SCHEMA) {
|
||||
final String parameterName = p.getName();
|
||||
IConnection connection = null;
|
||||
final List<? extends IConnection> connections = NodeUtil.getIncomingConnections(node, p.getContext());
|
||||
if(connections != null && !connections.isEmpty()) {
|
||||
connection = connections.get(0);
|
||||
}
|
||||
if(connection != null) {
|
||||
IMetadataTable metaTable = connection.getMetadataTable();
|
||||
List<IMetadataColumn> columns = metaTable.getListColumns();
|
||||
for(int i = 0; i < columns.size(); i++) {
|
||||
IMetadataColumn column = columns.get(i);
|
||||
%>
|
||||
component_parameters.put("<%=parameterName%>[<%=i%>]", "<%=column.getLabel()%>");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {
|
||||
final String key;
|
||||
if(!p.getName().contains("$")){
|
||||
key = p.getName();
|
||||
}else{
|
||||
final StringBuilder keyBuilder = new StringBuilder();
|
||||
for (String part : p.getName().split("\\.")) {
|
||||
if (keyBuilder.length() != 0) {
|
||||
keyBuilder.append(".");
|
||||
}
|
||||
if (part.contains("$") && !part.startsWith("$")) {
|
||||
keyBuilder.append(part.replace("$", "."));
|
||||
} else {
|
||||
keyBuilder.append(part);
|
||||
}
|
||||
}
|
||||
key = keyBuilder.toString();
|
||||
}
|
||||
String value = null;
|
||||
if(p.getFieldType() == EParameterFieldType.PASSWORD) {
|
||||
continue;
|
||||
} else {
|
||||
value = ElementParameterParser.getStringElementParameterValue(p);
|
||||
if (!org.talend.core.model.utils.ContextParameterUtils.isDynamic(value)) {
|
||||
value = org.talend.core.model.utils.TalendTextUtils.removeQuotes(value);
|
||||
value = org.talend.core.model.utils.TalendTextUtils.addQuotes(value);
|
||||
}
|
||||
}
|
||||
if (value != null) {
|
||||
if(key.endsWith("$maxBatchSize")){
|
||||
%>
|
||||
|
||||
<%
|
||||
} else if(p.getFieldType() == EParameterFieldType.CLOSED_LIST) {
|
||||
String valueTemp = org.talend.core.model.utils.TalendTextUtils.removeQuotes(value);
|
||||
if ("".equals(valueTemp)) {
|
||||
String[] listItemsDisplayCodeValue = p.getListItemsDisplayCodeName();
|
||||
if(listItemsDisplayCodeValue != null && listItemsDisplayCodeValue.length > 0){
|
||||
valueTemp = listItemsDisplayCodeValue[0];
|
||||
value = org.talend.core.model.utils.TalendTextUtils.addQuotes(valueTemp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(value==null || "null".equals(value.trim())) {
|
||||
value = "(Object)null";
|
||||
}
|
||||
%>
|
||||
component_parameters.put("<%=key%>", String.valueOf(<%=value%>));
|
||||
<%
|
||||
} // else do not put value in configuration
|
||||
}
|
||||
%>
|
||||
@@ -68,14 +68,6 @@
|
||||
id="org.talend.designer.components.model.UserComponentsProvider">
|
||||
</ComponentsProvider>
|
||||
</extension>
|
||||
<extension
|
||||
point="org.talend.core.components_provider">
|
||||
<ComponentsProvider
|
||||
class="org.talend.designer.codegen.components.model.SharedStudioUserComponentProvider"
|
||||
folderName="user"
|
||||
id="org.talend.designer.codegen.components.model.SharedStudioUserComponentProvider">
|
||||
</ComponentsProvider>
|
||||
</extension>
|
||||
<extension
|
||||
point="org.eclipse.core.runtime.preferences">
|
||||
<initializer
|
||||
|
||||
@@ -26,8 +26,10 @@ import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.ResourceBundle;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
@@ -73,7 +75,6 @@ import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.model.components.IComponentsHandler;
|
||||
import org.talend.core.model.components.filters.ComponentsFactoryProviderManager;
|
||||
import org.talend.core.model.components.filters.IComponentFactoryFilter;
|
||||
import org.talend.core.runtime.util.ComponentsLocationProvider;
|
||||
import org.talend.core.ui.IJobletProviderService;
|
||||
import org.talend.core.ui.ISparkJobletProviderService;
|
||||
import org.talend.core.ui.ISparkStreamingJobletProviderService;
|
||||
@@ -82,6 +83,8 @@ import org.talend.core.ui.images.CoreImageProvider;
|
||||
import org.talend.core.utils.TalendCacheUtils;
|
||||
import org.talend.designer.codegen.CodeGeneratorActivator;
|
||||
import org.talend.designer.codegen.i18n.Messages;
|
||||
import org.talend.designer.core.ITisLocalProviderService;
|
||||
import org.talend.designer.core.ITisLocalProviderService.ResClassLoader;
|
||||
import org.talend.designer.core.model.components.ComponentBundleToPath;
|
||||
import org.talend.designer.core.model.components.ComponentFilesNaming;
|
||||
import org.talend.designer.core.model.components.EmfComponent;
|
||||
@@ -161,11 +164,7 @@ public class ComponentsFactory implements IComponentsFactory {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
isInitialising.set(true);
|
||||
try {
|
||||
removeOldComponentsUserFolder();
|
||||
} catch (IOException ex) {
|
||||
ExceptionHandler.process(ex);
|
||||
} // not used anymore
|
||||
removeOldComponentsUserFolder(); // not used anymore
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// TimeMeasure.display = true;
|
||||
@@ -388,12 +387,10 @@ public class ComponentsFactory implements IComponentsFactory {
|
||||
ComponentManager.saveResource();
|
||||
}
|
||||
|
||||
private void removeOldComponentsUserFolder() throws IOException {
|
||||
private void removeOldComponentsUserFolder() {
|
||||
String userPath = IComponentsFactory.COMPONENTS_INNER_FOLDER + File.separatorChar
|
||||
+ ComponentUtilities.getExtFolder(OLD_COMPONENTS_USER_INNER_FOLDER);
|
||||
ComponentsProviderManager componentsProviderManager = ComponentsProviderManager.getInstance();
|
||||
AbstractComponentsProvider componentsProvider = componentsProviderManager.loadUserComponentsProvidersFromExtension();
|
||||
File componentsLocation = getComponentsLocation(componentsProvider, userPath);
|
||||
File componentsLocation = getComponentsLocation(userPath);
|
||||
if (componentsLocation != null && componentsLocation.exists()) {
|
||||
FilesUtils.removeFolder(componentsLocation, true);
|
||||
}
|
||||
@@ -674,38 +671,114 @@ public class ComponentsFactory implements IComponentsFactory {
|
||||
*
|
||||
* @param currentFolder
|
||||
* @return
|
||||
* @throws IOException
|
||||
* @throws BusinessException
|
||||
*/
|
||||
|
||||
private File getComponentsLocation(AbstractComponentsProvider componentsProvider, String folder) throws IOException {
|
||||
|
||||
if (componentsProvider instanceof ComponentsLocationProvider) {
|
||||
return componentsProvider.getInstallationFolder();
|
||||
} else {
|
||||
String componentsPath = IComponentsFactory.COMPONENTS_LOCATION;
|
||||
IBrandingService breaningService = (IBrandingService) GlobalServiceRegister.getDefault()
|
||||
.getService(IBrandingService.class);
|
||||
if (breaningService.isPoweredOnlyCamel()) {
|
||||
componentsPath = IComponentsFactory.CAMEL_COMPONENTS_LOCATION;
|
||||
}
|
||||
Bundle b = Platform.getBundle(componentsPath);
|
||||
private File getComponentsLocation(String folder) {
|
||||
String componentsPath = IComponentsFactory.COMPONENTS_LOCATION;
|
||||
IBrandingService breaningService = (IBrandingService) GlobalServiceRegister.getDefault()
|
||||
.getService(IBrandingService.class);
|
||||
if (breaningService.isPoweredOnlyCamel()) {
|
||||
componentsPath = IComponentsFactory.CAMEL_COMPONENTS_LOCATION;
|
||||
}
|
||||
Bundle b = Platform.getBundle(componentsPath);
|
||||
|
||||
File file = null;
|
||||
try {
|
||||
File file = null;
|
||||
try {
|
||||
URL url = FileLocator.find(b, new Path(folder), null);
|
||||
if (url == null) {
|
||||
return null;
|
||||
}
|
||||
URL fileUrl = FileLocator.toFileURL(url);
|
||||
file = new File(fileUrl.getPath());
|
||||
} catch (Exception e) {
|
||||
// e.printStackTrace();
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
private File getComponentsLocation(String folder, AbstractComponentsProvider provider) {
|
||||
File file = null;
|
||||
try {
|
||||
if (provider != null) {
|
||||
file = provider.getInstallationFolder();
|
||||
} else {
|
||||
String componentsPath = IComponentsFactory.COMPONENTS_LOCATION;
|
||||
Bundle b = Platform.getBundle(componentsPath);
|
||||
IBrandingService breaningService = (IBrandingService) GlobalServiceRegister.getDefault()
|
||||
.getService(IBrandingService.class);
|
||||
if (breaningService.isPoweredOnlyCamel()) {
|
||||
componentsPath = IComponentsFactory.CAMEL_COMPONENTS_LOCATION;
|
||||
}
|
||||
URL url = FileLocator.find(b, new Path(folder), null);
|
||||
if (url == null) {
|
||||
return null;
|
||||
}
|
||||
URL fileUrl = FileLocator.toFileURL(url);
|
||||
file = new File(fileUrl.getPath());
|
||||
} catch (Exception e) {
|
||||
// e.printStackTrace();
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
|
||||
return file;
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private ResourceBundle getComponentResourceBundle(IComponent currentComp, String source, String cachedPathSource,
|
||||
AbstractComponentsProvider provider) {
|
||||
try {
|
||||
AbstractComponentsProvider currentProvider = provider;
|
||||
if (currentProvider == null) {
|
||||
ComponentsProviderManager componentsProviderManager = ComponentsProviderManager.getInstance();
|
||||
Collection<AbstractComponentsProvider> providers = componentsProviderManager.getProviders();
|
||||
for (AbstractComponentsProvider curProvider : providers) {
|
||||
String path = new Path(curProvider.getInstallationFolder().toString()).toPortableString();
|
||||
if (source.startsWith(path)) {
|
||||
// fix for TDI-19889 and TDI-20507 to get the correct component provider
|
||||
if (cachedPathSource != null) {
|
||||
if (path.contains(cachedPathSource)) {
|
||||
currentProvider = curProvider;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
currentProvider = curProvider;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
String installPath = currentProvider.getInstallationFolder().toString();
|
||||
String label = ComponentFilesNaming.getInstance().getBundleName(currentComp.getName(),
|
||||
installPath.substring(installPath.lastIndexOf(IComponentsFactory.COMPONENTS_INNER_FOLDER)));
|
||||
|
||||
if (currentProvider.isUseLocalProvider()) {
|
||||
// if the component use local provider as storage (for user / ecosystem components)
|
||||
// then get the bundle resource from the current main component provider.
|
||||
|
||||
// note: code here to review later, service like this shouldn't be used...
|
||||
ResourceBundle bundle = null;
|
||||
IBrandingService brandingService = (IBrandingService) GlobalServiceRegister.getDefault()
|
||||
.getService(IBrandingService.class);
|
||||
if (brandingService.isPoweredOnlyCamel()) {
|
||||
bundle = currentProvider.getResourceBundle(label);
|
||||
} else {
|
||||
ITisLocalProviderService service = (ITisLocalProviderService) GlobalServiceRegister.getDefault()
|
||||
.getService(ITisLocalProviderService.class);
|
||||
bundle = service.getResourceBundle(label);
|
||||
}
|
||||
return bundle;
|
||||
} else {
|
||||
ResourceBundle bundle = ResourceBundle.getBundle(label, Locale.getDefault(),
|
||||
new ResClassLoader(currentProvider.getClass().getClassLoader()));
|
||||
return bundle;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private String getCodeLanguageSuffix() {
|
||||
@@ -1009,13 +1082,5 @@ public class ComponentsFactory implements IComponentsFactory {
|
||||
public void setComponentsHandler(IComponentsHandler componentsHandler) {
|
||||
this.componentsHandler = componentsHandler;
|
||||
}
|
||||
|
||||
public String getCustomComponentBundlePath() {
|
||||
ComponentsProviderManager componentsProviderManager = ComponentsProviderManager.getInstance();
|
||||
AbstractComponentsProvider componentsProvider = componentsProviderManager.loadUserComponentsProvidersFromExtension();
|
||||
String bundle = componentsProvider.getComponentsBundle();
|
||||
return ComponentBundleToPath.getPathFromBundle(bundle);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@ import org.eclipse.core.runtime.IExtensionRegistry;
|
||||
import org.eclipse.core.runtime.Platform;
|
||||
import org.talend.core.GlobalServiceRegister;
|
||||
import org.talend.core.model.components.AbstractComponentsProvider;
|
||||
import org.talend.core.runtime.util.SharedStudioInfoProvider;
|
||||
import org.talend.core.ui.branding.IBrandingService;
|
||||
import org.talend.designer.codegen.i18n.Messages;
|
||||
|
||||
@@ -70,9 +69,6 @@ public final class ComponentsProviderManager {
|
||||
try {
|
||||
AbstractComponentsProvider componentsProvider = (AbstractComponentsProvider) configurationElement
|
||||
.createExecutableExtension("class"); //$NON-NLS-1$
|
||||
if (componentsProvider instanceof SharedStudioInfoProvider && !((SharedStudioInfoProvider)componentsProvider).isSupportCurrentMode()) {
|
||||
continue;
|
||||
}
|
||||
componentsProvider.setId(id);
|
||||
componentsProvider.setFolderName(folderName);
|
||||
componentsProvider.setContributer(contributerName);
|
||||
@@ -85,15 +81,15 @@ public final class ComponentsProviderManager {
|
||||
}
|
||||
}
|
||||
|
||||
public AbstractComponentsProvider loadUserComponentsProvidersFromExtension() {
|
||||
if (providers == null) {
|
||||
loadComponentsProvidersFromExtension();
|
||||
}
|
||||
for (AbstractComponentsProvider provider : providers) {
|
||||
if (provider instanceof UserComponentsProvider) {
|
||||
return provider;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
public AbstractComponentsProvider loadUserComponentsProvidersFromExtension() {
|
||||
if (providers == null) {
|
||||
loadComponentsProvidersFromExtension();
|
||||
}
|
||||
for (AbstractComponentsProvider provider : providers) {
|
||||
if ("org.talend.designer.components.model.UserComponentsProvider".equals(provider.getId())) {
|
||||
return provider;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
package org.talend.designer.codegen.components.model;
|
||||
//============================================================================
|
||||
//
|
||||
//Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
//
|
||||
//This source code is available under agreement available at
|
||||
//%InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
//
|
||||
//You should have received a copy of the agreement
|
||||
//along with this program; if not, write to Talend SA
|
||||
//9 rue Pages 92150 Suresnes, France
|
||||
//
|
||||
//============================================================================
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.util.ResourceBundle;
|
||||
|
||||
import org.eclipse.core.runtime.IPath;
|
||||
import org.eclipse.core.runtime.Path;
|
||||
import org.eclipse.core.runtime.Platform;
|
||||
import org.talend.core.model.components.ComponentUtilities;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.runtime.util.ComponentsLocationProvider;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.designer.core.model.components.ComponentBundleToPath;
|
||||
|
||||
public class SharedStudioUserComponentProvider extends UserComponentsProvider implements ComponentsLocationProvider{
|
||||
@Override
|
||||
public File getInstallationFolder() throws IOException {
|
||||
File componentFolder = SharedStudioUtils.getSharedStudioComponentsParentFolder();
|
||||
IPath path = new Path(IComponentsFactory.COMPONENTS_INNER_FOLDER);
|
||||
path = path.append(IComponentsFactory.EXTERNAL_COMPONENTS_INNER_FOLDER).append(ComponentUtilities.getExtFolder(getFolderName()));
|
||||
File installationFolder = new File (componentFolder, path.toOSString());
|
||||
return installationFolder;
|
||||
}
|
||||
|
||||
|
||||
public String getComponentsBundle() {
|
||||
return ComponentBundleToPath.SHARED_STUDIO_CUSTOM_COMPONENT_BUNDLE;
|
||||
}
|
||||
|
||||
public boolean isSupportCurrentMode() {
|
||||
if (SharedStudioUtils.isSharedStudioMode()) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResourceBundle getResourceBundle(String label) {
|
||||
URL configFolderUrl = Platform.getConfigurationLocation().getURL();
|
||||
URLClassLoader urlLoader = new URLClassLoader(new java.net.URL[]{configFolderUrl});
|
||||
java.util.ResourceBundle bundle = java.util.ResourceBundle.getBundle( label ,
|
||||
java.util.Locale.getDefault(), urlLoader );
|
||||
return bundle;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -34,15 +34,13 @@ import org.talend.core.model.components.ComponentUtilities;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.model.general.Project;
|
||||
import org.talend.core.model.repository.ERepositoryObjectType;
|
||||
import org.talend.core.runtime.util.SharedStudioInfoProvider;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.core.ui.branding.IBrandingService;
|
||||
import org.talend.designer.codegen.CodeGeneratorActivator;
|
||||
import org.talend.designer.codegen.components.ui.IComponentPreferenceConstant;
|
||||
import org.talend.repository.ProjectManager;
|
||||
|
||||
/***/
|
||||
public class UserComponentsProvider extends AbstractCustomComponentsProvider implements SharedStudioInfoProvider{
|
||||
public class UserComponentsProvider extends AbstractCustomComponentsProvider {
|
||||
|
||||
@Override
|
||||
protected File getExternalComponentsLocation() {
|
||||
@@ -149,11 +147,5 @@ public class UserComponentsProvider extends AbstractCustomComponentsProvider imp
|
||||
public String getComponentsBundle() {
|
||||
return IComponentsFactory.COMPONENTS_LOCATION;
|
||||
}
|
||||
|
||||
public boolean isSupportCurrentMode() {
|
||||
if (SharedStudioUtils.isSharedStudioMode()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@ import java.util.Map;
|
||||
import org.eclipse.core.runtime.Platform;
|
||||
import org.talend.commons.exception.ExceptionHandler;
|
||||
import org.talend.commons.utils.StringUtils;
|
||||
import org.talend.designer.core.model.components.ComponentBundleToPath;
|
||||
|
||||
/**
|
||||
* Jet container for a particular component.
|
||||
@@ -214,17 +213,8 @@ public class JetBean {
|
||||
if (pluginIdToBundle.containsKey(pluginId)) {
|
||||
base = pluginIdToBundle.get(pluginId);
|
||||
} else {
|
||||
if (ComponentBundleToPath.SHARED_STUDIO_CUSTOM_COMPONENT_BUNDLE.equals(pluginId)) {
|
||||
base = ComponentBundleToPath.getPathFromBundle(pluginId);
|
||||
if (!base.endsWith("/")) {
|
||||
base = base + "/";
|
||||
}
|
||||
pluginIdToBundle.put(pluginId, base);
|
||||
} else {
|
||||
base = Platform.getBundle(pluginId).getEntry("/").toString(); //$NON-NLS-1$
|
||||
pluginIdToBundle.put(pluginId, base);
|
||||
}
|
||||
|
||||
base = Platform.getBundle(pluginId).getEntry("/").toString(); //$NON-NLS-1$
|
||||
pluginIdToBundle.put(pluginId, base);
|
||||
}
|
||||
String result = base + relativeUri;
|
||||
return result;
|
||||
|
||||
@@ -136,11 +136,13 @@ public class TalendJETCompiler extends JETCompiler {
|
||||
// get the plugin name from fileURI
|
||||
String refPluginName = matcher.group(1);
|
||||
// retrieve the plugin URI by pluginName.
|
||||
String realURI = TemplateUtil.getPlatformUrlOfBundle(refPluginName);
|
||||
if (realURI != null) {
|
||||
Bundle refBundle = Platform.getBundle(refPluginName);
|
||||
if (refBundle != null) {
|
||||
String realURI = TemplateUtil.getPlatformUrlOfBundle(refPluginName);
|
||||
// replace the old fileURI to new one by pluginURI
|
||||
String newFileURI = fileURI.replaceFirst(PLUGIN_VAR_PATTERN.pattern(), realURI);
|
||||
return newFileURI;
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@ package org.talend.designer.codegen.config;
|
||||
|
||||
import org.eclipse.core.runtime.Platform;
|
||||
import org.osgi.framework.Bundle;
|
||||
import org.talend.designer.core.model.components.ComponentBundleToPath;
|
||||
|
||||
/**
|
||||
* CodeGenerator Templates Ressources Utils.
|
||||
@@ -162,25 +161,10 @@ public class TemplateUtil {
|
||||
* @return
|
||||
*/
|
||||
public static String getPlatformUrlOfBundle(String bundleName) {
|
||||
if (ComponentBundleToPath.SHARED_STUDIO_CUSTOM_COMPONENT_BUNDLE.equals(bundleName)) {
|
||||
String basePath = ComponentBundleToPath.getPathFromBundle(bundleName);
|
||||
if (!basePath.endsWith("/")) {
|
||||
basePath = basePath + "/";
|
||||
}
|
||||
return basePath;
|
||||
} else {
|
||||
Bundle bundle = Platform.getBundle(bundleName);
|
||||
if (bundle == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("platform:/plugin/");
|
||||
sb.append(bundle.getSymbolicName());
|
||||
sb.append("_");
|
||||
sb.append(bundle.getVersion().toString());
|
||||
sb.append("/");
|
||||
return sb.toString();
|
||||
}
|
||||
Bundle bundle = Platform.getBundle(bundleName);
|
||||
if (bundle == null) {
|
||||
return null;
|
||||
}
|
||||
return "platform:/plugin/" + bundle.getSymbolicName() + "_" + bundle.getVersion().toString() + "/";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,7 +47,6 @@ import org.talend.core.ui.component.ComponentsFactoryProvider;
|
||||
import org.talend.designer.codegen.CodeGeneratorActivator;
|
||||
import org.talend.designer.codegen.config.TemplateUtil;
|
||||
import org.talend.designer.codegen.i18n.Messages;
|
||||
import org.talend.designer.core.model.components.ComponentBundleToPath;
|
||||
|
||||
/**
|
||||
* DOC xtan
|
||||
@@ -257,9 +256,10 @@ public final class JetSkeletonManager {
|
||||
};
|
||||
|
||||
for (TemplateUtil template : CodeGeneratorInternalTemplatesFactoryProvider.getInstance().getTemplates()) {
|
||||
Bundle b = Platform.getBundle(template.getJetPluginRepository());
|
||||
URL resourcesUrl = null;
|
||||
try {
|
||||
resourcesUrl = FileLocator.toFileURL(ComponentBundleToPath.findComponentsBundleURL(template.getJetPluginRepository(), new Path(template.getTemplateRelativeUri()), null));
|
||||
resourcesUrl = FileLocator.toFileURL(FileLocator.find(b, new Path(template.getTemplateRelativeUri()), null));
|
||||
} catch (IOException e) {
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
|
||||
@@ -9,14 +9,6 @@
|
||||
id="org.talend.designer.components.exchange.ExchangeComponentsProvider">
|
||||
</ComponentsProvider>
|
||||
</extension>
|
||||
<extension
|
||||
point="org.talend.core.components_provider">
|
||||
<ComponentsProvider
|
||||
class="org.talend.designer.components.exchange.SharedStudioExchangeComponentsProvider"
|
||||
folderName="exchange"
|
||||
id="org.talend.designer.components.exchange.SharedStudioExchangeComponentsProvider">
|
||||
</ComponentsProvider>
|
||||
</extension>
|
||||
<extension
|
||||
point="org.talend.core.runtime.service">
|
||||
<Service
|
||||
|
||||
@@ -28,15 +28,13 @@ import org.talend.core.GlobalServiceRegister;
|
||||
import org.talend.core.model.components.AbstractComponentsProvider;
|
||||
import org.talend.core.model.components.ComponentUtilities;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.runtime.util.SharedStudioInfoProvider;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.core.ui.branding.IBrandingService;
|
||||
import org.talend.designer.components.exchange.util.ExchangeUtils;
|
||||
|
||||
/**
|
||||
* DOC hcyi class global comment. Detailled comment
|
||||
*/
|
||||
public class ExchangeComponentsProvider extends AbstractComponentsProvider implements SharedStudioInfoProvider{
|
||||
public class ExchangeComponentsProvider extends AbstractComponentsProvider {
|
||||
|
||||
/**
|
||||
* ExchangeComponentsProvider constructor.
|
||||
@@ -186,10 +184,4 @@ public class ExchangeComponentsProvider extends AbstractComponentsProvider imple
|
||||
return IComponentsFactory.COMPONENTS_LOCATION;
|
||||
}
|
||||
|
||||
public boolean isSupportCurrentMode() {
|
||||
if (SharedStudioUtils.isSharedStudioMode()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
package org.talend.designer.components.exchange;
|
||||
//============================================================================
|
||||
//
|
||||
//Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
//
|
||||
//This source code is available under agreement available at
|
||||
//%InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
//
|
||||
//You should have received a copy of the agreement
|
||||
//along with this program; if not, write to Talend SA
|
||||
//9 rue Pages 92150 Suresnes, France
|
||||
//
|
||||
//============================================================================
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.util.ResourceBundle;
|
||||
|
||||
import org.eclipse.core.runtime.IPath;
|
||||
import org.eclipse.core.runtime.Path;
|
||||
import org.eclipse.core.runtime.Platform;
|
||||
import org.talend.core.model.components.ComponentUtilities;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.runtime.util.ComponentsLocationProvider;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.designer.core.model.components.ComponentBundleToPath;
|
||||
|
||||
public class SharedStudioExchangeComponentsProvider extends ExchangeComponentsProvider implements ComponentsLocationProvider{
|
||||
@Override
|
||||
public File getInstallationFolder() throws IOException {
|
||||
File componentFolder = SharedStudioUtils.getSharedStudioComponentsParentFolder();
|
||||
IPath path = new Path(IComponentsFactory.COMPONENTS_INNER_FOLDER);
|
||||
path = path.append(IComponentsFactory.EXTERNAL_COMPONENTS_INNER_FOLDER).append(ComponentUtilities.getExtFolder(getFolderName()));
|
||||
File installationFolder = new File (componentFolder, path.toOSString());
|
||||
return installationFolder;
|
||||
}
|
||||
|
||||
|
||||
public String getComponentsBundle() {
|
||||
return ComponentBundleToPath.SHARED_STUDIO_CUSTOM_COMPONENT_BUNDLE;
|
||||
}
|
||||
|
||||
public boolean isSupportCurrentMode() {
|
||||
if (SharedStudioUtils.isSharedStudioMode()) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResourceBundle getResourceBundle(String label) {
|
||||
URL configFolderUrl = Platform.getConfigurationLocation().getURL();
|
||||
URLClassLoader urlLoader = new URLClassLoader(new java.net.URL[]{configFolderUrl});
|
||||
java.util.ResourceBundle bundle = java.util.ResourceBundle.getBundle( label ,
|
||||
java.util.Locale.getDefault(), urlLoader );
|
||||
return bundle;
|
||||
}
|
||||
}
|
||||
@@ -52,7 +52,6 @@ import org.talend.core.download.IDownloadHelper;
|
||||
import org.talend.core.model.components.ComponentManager;
|
||||
import org.talend.core.model.components.IComponent;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.core.ui.component.ComponentPaletteUtilities;
|
||||
import org.talend.core.ui.component.ComponentsFactoryProvider;
|
||||
import org.talend.designer.codegen.ICodeGeneratorService;
|
||||
@@ -313,54 +312,51 @@ public class DownloadComponenentsAction extends Action implements IIntroAction {
|
||||
|
||||
protected void afterDownload(IProgressMonitor monitor, ComponentExtension extension, File localZipFile) throws Exception {
|
||||
if (UpdatesHelper.isComponentUpdateSite(localZipFile)) {
|
||||
if (!SharedStudioUtils.isSharedStudioMode()) {
|
||||
final File workFolder = org.talend.utils.files.FileUtils.createTmpFolder("downloadedComponents", ""); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
final File workFolder = org.talend.utils.files.FileUtils.createTmpFolder("downloadedComponents", ""); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
|
||||
try {
|
||||
FilesUtils.copyFile(localZipFile, new File(workFolder, localZipFile.getName()));
|
||||
try {
|
||||
FilesUtils.copyFile(localZipFile, new File(workFolder, localZipFile.getName()));
|
||||
|
||||
ComponentsInstallComponent component = LocalComponentInstallHelper.getComponent();
|
||||
if (component != null) {
|
||||
try {
|
||||
component.setComponentFolder(workFolder);
|
||||
if (component.install()) {
|
||||
ComponentsInstallComponent component = LocalComponentInstallHelper.getComponent();
|
||||
if (component != null) {
|
||||
try {
|
||||
component.setComponentFolder(workFolder);
|
||||
if (component.install()) {
|
||||
|
||||
|
||||
if (component.needRelaunch()) {
|
||||
askReboot();
|
||||
} else {
|
||||
MessageDialog.openInformation(DisplayUtils.getDefaultShell(),
|
||||
Messages.getString("DownloadComponenentsAction.installComponentsTitle"),
|
||||
component.getInstalledMessages());
|
||||
}
|
||||
} else {// install failure
|
||||
MessageDialog.openWarning(DisplayUtils.getDefaultShell(),
|
||||
Messages.getString("DownloadComponenentsAction_failureTitle"), //$NON-NLS-1$
|
||||
Messages.getString("DownloadComponenentsAction_failureMessage", extension.getLabel())); //$NON-NLS-1$
|
||||
if (component.needRelaunch()) {
|
||||
askReboot();
|
||||
} else {
|
||||
MessageDialog.openInformation(DisplayUtils.getDefaultShell(),
|
||||
Messages.getString("DownloadComponenentsAction.installComponentsTitle"),
|
||||
component.getInstalledMessages());
|
||||
}
|
||||
} finally {
|
||||
// after install, clear the setting for service.
|
||||
component.setComponentFolder(null);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// Popup dialog to user to waring install failed.
|
||||
Display.getDefault().syncExec(new Runnable() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
MessageDialog.openError(DisplayUtils.getDefaultShell(false),
|
||||
} else {// install failure
|
||||
MessageDialog.openWarning(DisplayUtils.getDefaultShell(),
|
||||
Messages.getString("DownloadComponenentsAction_failureTitle"), //$NON-NLS-1$
|
||||
Messages.getString("DownloadComponenentsAction_failureMessage", extension.getLabel())); //$NON-NLS-1$
|
||||
}
|
||||
});
|
||||
throw e;
|
||||
} finally {
|
||||
FilesUtils.deleteFolder(workFolder, true);
|
||||
} finally {
|
||||
// after install, clear the setting for service.
|
||||
component.setComponentFolder(null);
|
||||
}
|
||||
}
|
||||
monitor.done();
|
||||
ExchangeManager.getInstance().saveDownloadedExtensionsToFile(extension);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// Popup dialog to user to waring install failed.
|
||||
Display.getDefault().syncExec(new Runnable() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
MessageDialog.openError(DisplayUtils.getDefaultShell(false),
|
||||
Messages.getString("DownloadComponenentsAction_failureTitle"), //$NON-NLS-1$
|
||||
Messages.getString("DownloadComponenentsAction_failureMessage", extension.getLabel())); //$NON-NLS-1$
|
||||
}
|
||||
});
|
||||
throw e;
|
||||
} finally {
|
||||
FilesUtils.deleteFolder(workFolder, true);
|
||||
}
|
||||
monitor.done();
|
||||
ExchangeManager.getInstance().saveDownloadedExtensionsToFile(extension);
|
||||
} else {
|
||||
File installedLocation = ComponentInstaller.unzip(localZipFile.getAbsolutePath(), getComponentsFolder()
|
||||
.getAbsolutePath());
|
||||
|
||||
@@ -37,7 +37,6 @@ import org.eclipse.swt.widgets.Shell;
|
||||
import org.eclipse.ui.PlatformUI;
|
||||
import org.talend.commons.ui.runtime.exception.ExceptionHandler;
|
||||
import org.talend.core.download.DownloadHelper;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.designer.components.exchange.i18n.Messages;
|
||||
import org.talend.designer.components.exchange.model.Category;
|
||||
import org.talend.designer.components.exchange.model.VersionRevision;
|
||||
@@ -106,7 +105,7 @@ public class ImportExchangeDialog extends Dialog {
|
||||
|
||||
@Override
|
||||
protected void okPressed() {
|
||||
IPath tempPath = SharedStudioUtils.getTempFolderPath();
|
||||
IPath tempPath = new Path(System.getProperty("user.dir")).append("temp"); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
File pathFile = tempPath.toFile();
|
||||
if (downloadproperty.getFileName() == null || downloadproperty.getFileName() == null) {
|
||||
MessageBox box = new MessageBox(Display.getCurrent().getActiveShell(), SWT.ICON_WARNING | SWT.OK);
|
||||
|
||||
@@ -55,7 +55,6 @@ import org.talend.core.language.ECodeLanguage;
|
||||
import org.talend.core.language.LanguageManager;
|
||||
import org.talend.core.model.components.IComponentsFactory;
|
||||
import org.talend.core.model.general.Project;
|
||||
import org.talend.core.runtime.util.SharedStudioUtils;
|
||||
import org.talend.core.ui.component.ComponentPaletteUtilities;
|
||||
import org.talend.core.ui.component.ComponentsFactoryProvider;
|
||||
import org.talend.designer.components.exchange.ExchangePlugin;
|
||||
@@ -206,19 +205,14 @@ public class ExchangeUtils {
|
||||
* @return
|
||||
*/
|
||||
public static File getComponentFolder(String componentfolder) {
|
||||
if (SharedStudioUtils.isSharedStudioMode()) {
|
||||
File componentFolder = SharedStudioUtils.getSharedStudioComponentsExtFolder();
|
||||
return new File (componentFolder, componentfolder);
|
||||
} else {
|
||||
URL url = FileLocator.find(ExchangePlugin.getDefault().getBundle(), new Path(componentfolder), null);
|
||||
try {
|
||||
URL fileUrl = FileLocator.toFileURL(url);
|
||||
return new File(fileUrl.getPath());
|
||||
} catch (Exception e) {
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
URL url = FileLocator.find(ExchangePlugin.getDefault().getBundle(), new Path(componentfolder), null);
|
||||
try {
|
||||
URL fileUrl = FileLocator.toFileURL(url);
|
||||
return new File(fileUrl.getPath());
|
||||
} catch (Exception e) {
|
||||
ExceptionHandler.process(e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
|
||||
<!-- modification 2: compile classpath -->
|
||||
<path id="compile.classpath">
|
||||
<pathelement location="../../../../../../tcommon-studio-se/main/plugins/org.talend.libraries.dom4j-jaxen/lib/dom4j-2.1.3.jar" />
|
||||
<pathelement location="../../../../../../tcommon-studio-se/main/plugins/org.talend.libraries.dom4j-jaxen/lib/dom4j-1.6.1.jar" />
|
||||
<pathelement location="../../../../../../tcommon-studio-se/main/plugins/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" />
|
||||
</path>
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
<project name="org.talend.designer.components.libs" default="buildall" basedir=".">
|
||||
|
||||
<target name="buildall">
|
||||
<ant antfile="talend_file_enhanced_20070724/build.xml" target="process" inheritall="no" />
|
||||
<ant antfile="sugarCRMManagement/build.xml" target="process" inheritall="no" />
|
||||
<ant antfile="TalendSAX/build.xml" target="process" inheritall="no" />
|
||||
</target>
|
||||
|
||||
@@ -3,32 +3,11 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>commons-net-ftps-proxy</artifactId>
|
||||
<version>3.6.1-talend-20200902</version>
|
||||
<version>3.6.1-talend-20190819</version>
|
||||
|
||||
<name>commons-net-talend</name>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<slf4.version>1.7.25</slf4.version>
|
||||
<lombok.version>1.18.12</lombok.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>${lombok.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>${slf4.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>commons-net</groupId>
|
||||
<artifactId>commons-net</artifactId>
|
||||
@@ -36,6 +15,10 @@
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
|
||||
@@ -11,10 +11,8 @@ import javax.net.ssl.SSLSession;
|
||||
import javax.net.ssl.SSLSessionContext;
|
||||
import javax.net.ssl.SSLSocket;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.net.ftp.FTPSClient;
|
||||
|
||||
@Slf4j
|
||||
public class SSLSessionReuseFTPSClient extends FTPSClient {
|
||||
|
||||
public SSLSessionReuseFTPSClient(boolean isImplicit, SSLContext context) {
|
||||
@@ -26,12 +24,6 @@ public class SSLSessionReuseFTPSClient extends FTPSClient {
|
||||
if (socket instanceof SSLSocket) {
|
||||
final SSLSession session = ((SSLSocket) _socket_).getSession();
|
||||
final SSLSessionContext context = session.getSessionContext();
|
||||
if (context == null) {
|
||||
// TDI-44654 (may be reproduced with Syncplify server)
|
||||
log.info("SSL Session Context is null. SSL Session was re-initialized.");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
final Field sessionHostPortCache = context.getClass().getDeclaredField("sessionHostPortCache");
|
||||
sessionHostPortCache.setAccessible(true);
|
||||
@@ -40,10 +32,10 @@ public class SSLSessionReuseFTPSClient extends FTPSClient {
|
||||
putMethod.setAccessible(true);
|
||||
InetAddress address = socket.getInetAddress();
|
||||
int port = socket.getPort();
|
||||
|
||||
|
||||
String key = String.format("%s:%s", address.getHostName(), String.valueOf(port)).toLowerCase(Locale.ROOT);
|
||||
putMethod.invoke(cache, key, session);
|
||||
|
||||
|
||||
key = String.format("%s:%s", address.getHostAddress(), String.valueOf(port)).toLowerCase(Locale.ROOT);
|
||||
putMethod.invoke(cache, key, session);
|
||||
} catch (Exception e) {
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>filecopy</artifactId>
|
||||
<version>2.0.1</version>
|
||||
<version>2.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend-copy</name>
|
||||
|
||||
@@ -15,10 +15,7 @@ package org.talend;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.nio.file.attribute.FileTime;
|
||||
|
||||
/**
|
||||
* DOC Administrator class global comment. Detailled comment
|
||||
@@ -38,17 +35,14 @@ public class FileCopy {
|
||||
* @throws IOException : if IO pb.
|
||||
*/
|
||||
public static void copyFile(String srcFileName, String desFileName, boolean delSrc) throws IOException {
|
||||
final Path source = Paths.get(srcFileName);
|
||||
final Path destination = Paths.get(desFileName);
|
||||
final File source = new File(srcFileName);
|
||||
final File destination = new File(desFileName);
|
||||
|
||||
if (delSrc) {
|
||||
// move : more efficient if in same FS and mustr delete existing file.
|
||||
FileTime lastModifiedTime = Files.getLastModifiedTime(source);
|
||||
Files.move(source, destination, StandardCopyOption.REPLACE_EXISTING);
|
||||
Files.setLastModifiedTime(destination,lastModifiedTime);
|
||||
Files.move(source.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
} else {
|
||||
Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING);
|
||||
Files.setLastModifiedTime(destination,Files.getLastModifiedTime(source));
|
||||
Files.copy(source.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -100,24 +100,6 @@ class FileCopyTest {
|
||||
Assertions.assertEquals(referenceSize, copy.length(), "Size error");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLastModifiedTime() throws Exception {
|
||||
final URL repCopy = Thread.currentThread().getContextClassLoader().getResource("copy");
|
||||
|
||||
File file = this.buildFile("fileLMT.txt", 10L * 1024L);
|
||||
file.deleteOnExit();
|
||||
long referencceTime = 324723894L;
|
||||
file.setLastModified(referencceTime);
|
||||
|
||||
File copy = new File(repCopy.getPath(), "fileLMTDestination.txt");
|
||||
if (copy.exists()) {
|
||||
copy.delete();
|
||||
}
|
||||
copy.deleteOnExit();
|
||||
FileCopy.copyFile(file.getPath(), copy.getPath(), true);
|
||||
Assertions.assertEquals(referencceTime, copy.lastModified(), "modified time is not idential");
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a new file for testing.
|
||||
*
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dom4j</groupId>
|
||||
<groupId>dom4j</groupId>
|
||||
<artifactId>dom4j</artifactId>
|
||||
<version>2.1.3</version>
|
||||
<version>1.6.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-lang</groupId>
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>simpleexcel</artifactId>
|
||||
<version>2.4-20200923</version>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>simpleexcel-2.2-20190722</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>simpleexcel</name>
|
||||
@@ -13,7 +13,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<java.source.version>1.8</java.source.version>
|
||||
<java.source.version>1.6</java.source.version>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
@@ -43,30 +43,48 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi</artifactId>
|
||||
<version>4.1.2</version>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-scratchpad</artifactId>
|
||||
<version>4.1.2</version>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml</artifactId>
|
||||
<version>4.1.2</version>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml-schemas -->
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml-schemas</artifactId>
|
||||
<version>4.1.2</version>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.geronimo.specs/geronimo-stax-api_1.0_spec -->
|
||||
<dependency>
|
||||
<groupId>org.apache.geronimo.specs</groupId>
|
||||
<artifactId>geronimo-stax-api_1.0_spec</artifactId>
|
||||
<version>1.0</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/dom4j/dom4j -->
|
||||
<dependency>
|
||||
<groupId>dom4j</groupId>
|
||||
<artifactId>dom4j</artifactId>
|
||||
<version>1.6.1</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.xmlbeans/xmlbeans -->
|
||||
<dependency>
|
||||
<groupId>org.apache.xmlbeans</groupId>
|
||||
<artifactId>xmlbeans</artifactId>
|
||||
<version>3.1.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<resources>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// ============================================================================
|
||||
//
|
||||
// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
|
||||
// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
//
|
||||
// This source code is available under agreement available at
|
||||
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// ============================================================================
|
||||
//
|
||||
// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
|
||||
// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
//
|
||||
// This source code is available under agreement available at
|
||||
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// ============================================================================
|
||||
//
|
||||
// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
|
||||
// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
//
|
||||
// This source code is available under agreement available at
|
||||
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// ============================================================================
|
||||
//
|
||||
// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
|
||||
// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
//
|
||||
// This source code is available under agreement available at
|
||||
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// ============================================================================
|
||||
//
|
||||
// Copyright (C) 2006-2020 Talend Inc. - www.talend.com
|
||||
// Copyright (C) 2006-2019 Talend Inc. - www.talend.com
|
||||
//
|
||||
// This source code is available under agreement available at
|
||||
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>components-soap</artifactId>
|
||||
<version>2.3-20200918</version>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend-soap</artifactId>
|
||||
<version>2.1-20190716</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend-soap</name>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<cxf.version>3.1.1</cxf.version>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
</properties>
|
||||
|
||||
@@ -45,24 +46,29 @@
|
||||
<systemPath>${java.home}/lib/rt.jar</systemPath>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dom4j</groupId>
|
||||
<artifactId>dom4j</artifactId>
|
||||
<version>2.1.3</version>
|
||||
<groupId>jdom</groupId>
|
||||
<artifactId>jdom</artifactId>
|
||||
<version>1.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.sun.xml.messaging.saaj</groupId>
|
||||
<artifactId>saaj-impl</artifactId>
|
||||
<version>1.5.2</version>
|
||||
<groupId>com.sun.xml.messaging.saaj</groupId>
|
||||
<artifactId>saaj-impl</artifactId>
|
||||
<version>1.3.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>javax.activation</groupId>
|
||||
<artifactId>activation</artifactId>
|
||||
<version>1.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>xerces</groupId>
|
||||
<artifactId>xercesImpl</artifactId>
|
||||
<version>2.12.0</version>
|
||||
<version>2.6.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<version>1.14</version>
|
||||
<version>1.9</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
@@ -102,4 +108,4 @@
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
</project>
|
||||
@@ -32,7 +32,8 @@ import javax.xml.transform.stream.StreamResult;
|
||||
import javax.xml.transform.stream.StreamSource;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.dom4j.io.DOMReader;
|
||||
import org.jdom.input.DOMBuilder;
|
||||
import org.jdom.output.XMLOutputter;
|
||||
import org.talend.soap.sun.SunNtlmAuthenticationUpdater;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
@@ -44,6 +45,8 @@ public class SOAPUtil {
|
||||
|
||||
private static final String vmVendor = System.getProperty("java.vendor.url");
|
||||
|
||||
private static final String ibmVmVendor = "http://www.ibm.com/";
|
||||
|
||||
private static final String sunVmVendor = "http://java.sun.com/";
|
||||
|
||||
private static final String oracleVmVendor = "http://java.oracle.com/";
|
||||
@@ -137,7 +140,12 @@ public class SOAPUtil {
|
||||
StreamSource preppedMsgSrc = new StreamSource(stream);
|
||||
soapPart.setContent(preppedMsgSrc);
|
||||
|
||||
// InputStream stream = new FileInputStream(new File("d://soap.txt"));
|
||||
// StreamSource preppedMsgSrc = new StreamSource(stream);
|
||||
// soapPart.setContent(preppedMsgSrc);
|
||||
|
||||
message.saveChanges();
|
||||
// Send the message
|
||||
|
||||
SOAPMessage reply = connection.call(message, destination);
|
||||
|
||||
@@ -218,7 +226,7 @@ public class SOAPUtil {
|
||||
Node content;
|
||||
Element headerRootElem = document.createElement("Header");
|
||||
|
||||
Iterator<javax.xml.soap.Node> childElements = header.getChildElements();
|
||||
Iterator childElements = header.getChildElements();
|
||||
org.w3c.dom.Node domNode = null;
|
||||
while (childElements.hasNext()) {
|
||||
domNode = (org.w3c.dom.Node) childElements.next();
|
||||
@@ -237,11 +245,12 @@ public class SOAPUtil {
|
||||
return reHeaderMessage;
|
||||
}
|
||||
|
||||
private String Doc2StringWithoutDeclare(Document doc) {
|
||||
DOMReader reader = new DOMReader();
|
||||
org.dom4j.Document document = reader.read(doc);
|
||||
return document.getRootElement().asXML();
|
||||
}
|
||||
private String Doc2StringWithoutDeclare(Document doc) {
|
||||
DOMBuilder builder = new DOMBuilder();
|
||||
org.jdom.Document jdomDoc = builder.build(doc);
|
||||
XMLOutputter outputter = new XMLOutputter();
|
||||
return outputter.outputString(jdomDoc.getRootElement());
|
||||
}
|
||||
|
||||
/**
|
||||
* invoke soap and return the response document
|
||||
@@ -354,4 +363,4 @@ public class SOAPUtil {
|
||||
headers.setHeader("Authorization", "Basic " + encodeUserInfo);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>talend-aws</artifactId>
|
||||
<version>1.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talend-aws</name>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<java.source.version>1.8</java.source.version>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
</snapshotRepository>
|
||||
<repository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk</artifactId>
|
||||
<version>1.11.848</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/java</directory>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>${java.source.version}</source>
|
||||
<target>${java.source.version}</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@@ -1,277 +0,0 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import static com.amazonaws.event.SDKProgressPublisher.publishProgress;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedList;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import com.amazonaws.AmazonClientException;
|
||||
import com.amazonaws.AmazonServiceException;
|
||||
import com.amazonaws.event.ProgressEventType;
|
||||
import com.amazonaws.event.ProgressListener;
|
||||
import com.amazonaws.event.ProgressListenerChain;
|
||||
import com.amazonaws.services.s3.model.LegacyS3ProgressListener;
|
||||
import com.amazonaws.services.s3.transfer.Transfer;
|
||||
import com.amazonaws.services.s3.transfer.TransferProgress;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferMonitor;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
|
||||
|
||||
|
||||
/**
|
||||
* Abstract transfer implementation.
|
||||
*/
|
||||
public abstract class AbstractTransfer implements Transfer {
|
||||
|
||||
/** The current state of this transfer. */
|
||||
protected volatile TransferState state = TransferState.Waiting;
|
||||
|
||||
protected TransferMonitor monitor;
|
||||
|
||||
/** The progress of this transfer. */
|
||||
private final TransferProgress transferProgress;
|
||||
|
||||
private final String description;
|
||||
|
||||
/** Hook for adding/removing more progress listeners. */
|
||||
protected final ProgressListenerChain listenerChain;
|
||||
|
||||
/** Collection of listeners to be notified for changes to the state of this transfer via setState() */
|
||||
protected final Collection<TransferStateChangeListener> stateChangeListeners = new LinkedList<TransferStateChangeListener>();
|
||||
|
||||
AbstractTransfer(String description, TransferProgress transferProgress, ProgressListenerChain progressListenerChain) {
|
||||
this(description, transferProgress, progressListenerChain, null);
|
||||
}
|
||||
|
||||
AbstractTransfer(String description, TransferProgress transferProgress,
|
||||
ProgressListenerChain progressListenerChain, TransferStateChangeListener stateChangeListener) {
|
||||
this.description = description;
|
||||
this.listenerChain = progressListenerChain;
|
||||
this.transferProgress = transferProgress;
|
||||
addStateChangeListener(stateChangeListener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether or not the transfer is finished (i.e. completed successfully,
|
||||
* failed, or was canceled). This method should never block.
|
||||
*
|
||||
* @return Returns <code>true</code> if this transfer is finished (i.e. completed successfully,
|
||||
* failed, or was canceled). Returns <code>false</code> if otherwise.
|
||||
*/
|
||||
public final synchronized boolean isDone() {
|
||||
return (state == TransferState.Failed ||
|
||||
state == TransferState.Completed ||
|
||||
state == TransferState.Canceled);
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for this transfer to complete. This is a blocking call; the current
|
||||
* thread is suspended until this transfer completes.
|
||||
*
|
||||
* @throws AmazonClientException
|
||||
* If any errors were encountered in the client while making the
|
||||
* request or handling the response.
|
||||
* @throws AmazonServiceException
|
||||
* If any errors occurred in Amazon S3 while processing the
|
||||
* request.
|
||||
* @throws InterruptedException
|
||||
* If this thread is interrupted while waiting for the transfer
|
||||
* to complete.
|
||||
*/
|
||||
public void waitForCompletion()
|
||||
throws AmazonClientException, AmazonServiceException, InterruptedException {
|
||||
try {
|
||||
Object result = null;
|
||||
while (!monitor.isDone() || result == null) {
|
||||
Future<?> f = monitor.getFuture();
|
||||
result = f.get();
|
||||
}
|
||||
} catch (ExecutionException e) {
|
||||
rethrowExecutionException(e);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for this transfer to finish and returns any error that occurred, or
|
||||
* returns <code>null</code> if no errors occurred.
|
||||
* This is a blocking call; the current thread
|
||||
* will be suspended until this transfer either fails or completes
|
||||
* successfully.
|
||||
*
|
||||
* @return Any error that occurred while processing this transfer.
|
||||
* Otherwise returns <code>null</code> if no errors occurred.
|
||||
*
|
||||
* @throws InterruptedException
|
||||
* If this thread is interrupted while waiting for the transfer
|
||||
* to complete.
|
||||
*/
|
||||
public AmazonClientException waitForException() throws InterruptedException {
|
||||
try {
|
||||
|
||||
/**
|
||||
* Do not remove the while loop. We need this as the future returned by
|
||||
* monitor.getFuture() is set two times during the upload and copy operations.
|
||||
*/
|
||||
while (!monitor.isDone()) {
|
||||
monitor.getFuture().get();
|
||||
}
|
||||
monitor.getFuture().get();
|
||||
return null;
|
||||
} catch (ExecutionException e) {
|
||||
return unwrapExecutionException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a human-readable description of this transfer.
|
||||
*
|
||||
* @return A human-readable description of this transfer.
|
||||
*/
|
||||
public String getDescription() {
|
||||
return description;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current state of this transfer.
|
||||
*
|
||||
* @return The current state of this transfer.
|
||||
*/
|
||||
public synchronized TransferState getState() {
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the current state of this transfer.
|
||||
*/
|
||||
public void setState(TransferState state) {
|
||||
synchronized (this) {
|
||||
this.state = state;
|
||||
}
|
||||
for ( TransferStateChangeListener listener : stateChangeListeners ) {
|
||||
listener.transferStateChanged(this, state);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Notifies all the registered state change listeners of the state update.
|
||||
*/
|
||||
public void notifyStateChangeListeners(TransferState state) {
|
||||
for ( TransferStateChangeListener listener : stateChangeListeners ) {
|
||||
listener.transferStateChanged(this, state);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the specified progress listener to the list of listeners
|
||||
* receiving updates about this transfer's progress.
|
||||
*
|
||||
* @param listener
|
||||
* The progress listener to add.
|
||||
*/
|
||||
public synchronized void addProgressListener(ProgressListener listener) {
|
||||
listenerChain.addProgressListener(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the specified progress listener from the list of progress
|
||||
* listeners receiving updates about this transfer's progress.
|
||||
*
|
||||
* @param listener
|
||||
* The progress listener to remove.
|
||||
*/
|
||||
public synchronized void removeProgressListener(ProgressListener listener) {
|
||||
listenerChain.removeProgressListener(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Replaced by {@link #addProgressListener(ProgressListener)}
|
||||
*/
|
||||
@Deprecated
|
||||
public synchronized void addProgressListener(com.amazonaws.services.s3.model.ProgressListener listener) {
|
||||
listenerChain.addProgressListener(new LegacyS3ProgressListener(listener));
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Replaced by {@link #removeProgressListener(ProgressListener)}
|
||||
*/
|
||||
@Deprecated
|
||||
public synchronized void removeProgressListener(com.amazonaws.services.s3.model.ProgressListener listener) {
|
||||
listenerChain.removeProgressListener(new LegacyS3ProgressListener(listener));
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the given state change listener to the collection of listeners.
|
||||
*/
|
||||
public synchronized void addStateChangeListener(TransferStateChangeListener listener) {
|
||||
if ( listener != null )
|
||||
stateChangeListeners.add(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the given state change listener from the collection of listeners.
|
||||
*/
|
||||
public synchronized void removeStateChangeListener(TransferStateChangeListener listener) {
|
||||
if ( listener != null )
|
||||
stateChangeListeners.remove(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns progress information about this transfer.
|
||||
*
|
||||
* @return The progress information about this transfer.
|
||||
*/
|
||||
public TransferProgress getProgress() {
|
||||
return transferProgress;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the monitor used to poll for transfer completion.
|
||||
*/
|
||||
public void setMonitor(TransferMonitor monitor) {
|
||||
this.monitor = monitor;
|
||||
}
|
||||
|
||||
public TransferMonitor getMonitor() {
|
||||
return monitor;
|
||||
}
|
||||
|
||||
protected void fireProgressEvent(final ProgressEventType eventType) {
|
||||
publishProgress(listenerChain, eventType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Examines the cause of the specified ExecutionException and either
|
||||
* rethrows it directly (if it's a type of AmazonClientException) or wraps
|
||||
* it in an AmazonClientException and rethrows it.
|
||||
*
|
||||
* @param e
|
||||
* The execution exception to examine.
|
||||
*/
|
||||
protected void rethrowExecutionException(ExecutionException e) {
|
||||
throw unwrapExecutionException(e);
|
||||
}
|
||||
|
||||
/**
|
||||
* Unwraps the root exception that caused the specified ExecutionException
|
||||
* and returns it. If it was not an instance of AmazonClientException, it is
|
||||
* wrapped as an AmazonClientException.
|
||||
*
|
||||
* @param e
|
||||
* The ExecutionException to unwrap.
|
||||
*
|
||||
* @return The root exception that caused the specified ExecutionException.
|
||||
*/
|
||||
protected AmazonClientException unwrapExecutionException(ExecutionException e) {
|
||||
Throwable t = e;
|
||||
while (t.getCause() != null && t instanceof ExecutionException) {
|
||||
t = t.getCause();
|
||||
}
|
||||
if (t instanceof AmazonClientException) {
|
||||
return (AmazonClientException) t;
|
||||
}
|
||||
return new AmazonClientException("Unable to complete transfer: " + t.getMessage(), t);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.annotation.SdkInternalApi;
|
||||
import com.amazonaws.services.s3.internal.ServiceUtils;
|
||||
import com.amazonaws.services.s3.transfer.Transfer;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
/**
|
||||
* Helper class to merge all the individual part files into a destinationFile.
|
||||
*/
|
||||
@SdkInternalApi
|
||||
public class CompleteMultipartDownload implements Callable<File> {
|
||||
private final List<Future<File>> partFiles;
|
||||
private final File destinationFile;
|
||||
private final DownloadImpl download;
|
||||
private Integer currentPartNumber;
|
||||
|
||||
public CompleteMultipartDownload(List<Future<File>> files, File destinationFile, DownloadImpl download, Integer currentPartNumber) {
|
||||
this.partFiles = files;
|
||||
this.destinationFile = destinationFile;
|
||||
this.download = download;
|
||||
this.currentPartNumber = currentPartNumber;
|
||||
}
|
||||
|
||||
@Override
|
||||
public File call() throws Exception {
|
||||
for (Future<File> file : partFiles) {
|
||||
ServiceUtils.appendFile(file.get(), destinationFile);
|
||||
download.updatePersistableTransfer(currentPartNumber++);
|
||||
}
|
||||
|
||||
download.setState(Transfer.TransferState.Completed);
|
||||
return destinationFile;
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import com.amazonaws.services.s3.model.CryptoMode;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.transfer.Transfer;
|
||||
import com.amazonaws.services.s3.transfer.exception.PauseException;
|
||||
|
||||
/**
|
||||
* Represents an asynchronous download from Amazon S3.
|
||||
*/
|
||||
public interface Download extends Transfer {
|
||||
|
||||
/**
|
||||
* Returns the ObjectMetadata for the object being downloaded.
|
||||
*
|
||||
* @return The ObjectMetadata for the object being downloaded.
|
||||
*/
|
||||
public ObjectMetadata getObjectMetadata();
|
||||
|
||||
/**
|
||||
* The name of the bucket where the object is being downloaded from.
|
||||
*
|
||||
* @return The name of the bucket where the object is being downloaded from.
|
||||
*/
|
||||
public String getBucketName();
|
||||
|
||||
/**
|
||||
* The key under which this object was stored in Amazon S3.
|
||||
*
|
||||
* @return The key under which this object was stored in Amazon S3.
|
||||
*/
|
||||
public String getKey();
|
||||
|
||||
/**
|
||||
* Cancels this download.
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
public void abort() throws IOException;
|
||||
|
||||
/**
|
||||
* Pause the current download operation and returns the information that can
|
||||
* be used to resume the download at a later time.
|
||||
*
|
||||
* Resuming a download would not perform ETag check as range get is
|
||||
* performed for downloading the object's remaining contents.
|
||||
*
|
||||
* Resuming a download for an object encrypted using
|
||||
* {@link CryptoMode#StrictAuthenticatedEncryption} would result in
|
||||
* AmazonClientException as authenticity cannot be guaranteed for a range
|
||||
* get operation.
|
||||
*
|
||||
* @throws PauseException
|
||||
* If any errors were encountered while trying to pause the
|
||||
* download.
|
||||
*/
|
||||
public PersistableDownload pause() throws PauseException;
|
||||
}
|
||||
@@ -1,312 +0,0 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.net.SocketException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import javax.net.ssl.SSLProtocolException;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import com.amazonaws.AmazonClientException;
|
||||
import com.amazonaws.SdkClientException;
|
||||
import com.amazonaws.annotation.SdkInternalApi;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.internal.FileLocks;
|
||||
import com.amazonaws.services.s3.internal.ServiceUtils;
|
||||
import com.amazonaws.services.s3.internal.ServiceUtils.RetryableS3DownloadTask;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.amazonaws.services.s3.transfer.Transfer.TransferState;
|
||||
import com.amazonaws.services.s3.transfer.exception.FileLockException;
|
||||
import com.amazonaws.util.IOUtils;
|
||||
|
||||
@SdkInternalApi
|
||||
final class DownloadCallable implements Callable<File> {
|
||||
private static final Log LOG = LogFactory.getLog(DownloadCallable.class);
|
||||
|
||||
private final AmazonS3 s3;
|
||||
private final CountDownLatch latch;
|
||||
private final GetObjectRequest req;
|
||||
private final boolean resumeExistingDownload;
|
||||
private final DownloadImpl download;
|
||||
private final File dstfile;
|
||||
private final long origStartingByte;
|
||||
private final long timeout;
|
||||
private final ScheduledExecutorService timedExecutor;
|
||||
/** The thread pool in which parts are downloaded downloaded. */
|
||||
private final ExecutorService executor;
|
||||
private final List<Future<File>> futureFiles;
|
||||
private final boolean isDownloadParallel;
|
||||
private Integer lastFullyMergedPartNumber;
|
||||
private final boolean resumeOnRetry;
|
||||
|
||||
private long expectedFileLength;
|
||||
|
||||
DownloadCallable(AmazonS3 s3, CountDownLatch latch,
|
||||
GetObjectRequest req, boolean resumeExistingDownload,
|
||||
DownloadImpl download, File dstfile, long origStartingByte,
|
||||
long expectedFileLength, long timeout,
|
||||
ScheduledExecutorService timedExecutor,
|
||||
ExecutorService executor,
|
||||
Integer lastFullyDownloadedPartNumber, boolean isDownloadParallel, boolean resumeOnRetry)
|
||||
{
|
||||
if (s3 == null || latch == null || req == null || dstfile == null || download == null)
|
||||
throw new IllegalArgumentException();
|
||||
this.s3 = s3;
|
||||
this.latch = latch;
|
||||
this.req = req;
|
||||
this.resumeExistingDownload = resumeExistingDownload;
|
||||
this.download = download;
|
||||
this.dstfile = dstfile;
|
||||
this.origStartingByte = origStartingByte;
|
||||
this.expectedFileLength = expectedFileLength;
|
||||
this.timeout = timeout;
|
||||
this.timedExecutor = timedExecutor;
|
||||
this.executor = executor;
|
||||
this.futureFiles = new ArrayList<Future<File>>();
|
||||
this.lastFullyMergedPartNumber = lastFullyDownloadedPartNumber;
|
||||
this.isDownloadParallel = isDownloadParallel;
|
||||
this.resumeOnRetry = resumeOnRetry;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method must return a non-null object, or else the existing
|
||||
* implementation in {@link AbstractTransfer#waitForCompletion()}
|
||||
* would block forever.
|
||||
*
|
||||
* @return the downloaded file
|
||||
*/
|
||||
@Override
|
||||
public File call() throws Exception {
|
||||
try {
|
||||
latch.await();
|
||||
|
||||
if (isTimeoutEnabled()) {
|
||||
timedExecutor.schedule(new Runnable() {
|
||||
public void run() {
|
||||
try {
|
||||
if (download.getState() != TransferState.Completed) {
|
||||
download.abort();
|
||||
}
|
||||
} catch(Exception e) {
|
||||
throw new SdkClientException(
|
||||
"Unable to abort download after timeout", e);
|
||||
}
|
||||
}
|
||||
}, timeout, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
download.setState(TransferState.InProgress);
|
||||
ServiceUtils.createParentDirectoryIfNecessary(dstfile);
|
||||
|
||||
if (isDownloadParallel) {
|
||||
downloadInParallel(ServiceUtils.getPartCount(req, s3));
|
||||
} else {
|
||||
S3Object s3Object = retryableDownloadS3ObjectToFile(dstfile,
|
||||
new DownloadTaskImpl(s3, download, req));
|
||||
updateDownloadStatus(s3Object);
|
||||
}
|
||||
return dstfile;
|
||||
} catch (Throwable t) {
|
||||
// Cancel all the futures
|
||||
for (Future<File> f : futureFiles) {
|
||||
f.cancel(true);
|
||||
}
|
||||
// Downloads aren't allowed to move from canceled to failed
|
||||
if (download.getState() != TransferState.Canceled) {
|
||||
download.setState(TransferState.Failed);
|
||||
}
|
||||
if (t instanceof Exception)
|
||||
throw (Exception) t;
|
||||
else
|
||||
throw (Error) t;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes the result from serial download,
|
||||
* updates the transfer state and monitor in downloadImpl object
|
||||
* based on the result.
|
||||
*/
|
||||
private void updateDownloadStatus(S3Object result) {
|
||||
if (result == null) {
|
||||
download.setState(TransferState.Canceled);
|
||||
download.setMonitor(new DownloadMonitor(download, null));
|
||||
} else {
|
||||
download.setState(TransferState.Completed);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads each part of the object into a separate file synchronously and
|
||||
* combines all the files into a single file.
|
||||
*/
|
||||
private void downloadInParallel(int partCount) throws Exception {
|
||||
if (lastFullyMergedPartNumber == null) {
|
||||
lastFullyMergedPartNumber = 0;
|
||||
}
|
||||
|
||||
for (int i = lastFullyMergedPartNumber + 1; i <= partCount; i++) {
|
||||
GetObjectRequest getPartRequest = new GetObjectRequest(req.getBucketName(), req.getKey(),
|
||||
req.getVersionId()).withUnmodifiedSinceConstraint(req.getUnmodifiedSinceConstraint())
|
||||
.withModifiedSinceConstraint(req.getModifiedSinceConstraint())
|
||||
.withResponseHeaders(req.getResponseHeaders()).withSSECustomerKey(req.getSSECustomerKey())
|
||||
.withGeneralProgressListener(req.getGeneralProgressListener());
|
||||
|
||||
getPartRequest.setMatchingETagConstraints(req.getMatchingETagConstraints());
|
||||
getPartRequest.setNonmatchingETagConstraints(req.getNonmatchingETagConstraints());
|
||||
getPartRequest.setRequesterPays(req.isRequesterPays());
|
||||
|
||||
futureFiles.add(
|
||||
executor.submit(new DownloadPartCallable(s3, getPartRequest.withPartNumber(i), dstfile)));
|
||||
}
|
||||
|
||||
truncateDestinationFileIfNecessary();
|
||||
Future<File> future = executor.submit(new CompleteMultipartDownload(futureFiles, dstfile, download, ++lastFullyMergedPartNumber));
|
||||
((DownloadMonitor) download.getMonitor()).setFuture(future);
|
||||
}
|
||||
|
||||
/**
|
||||
* If only partial part object is merged into the dstFile(due to pause
|
||||
* operation), adjust the file length so that the part starts writing from
|
||||
* the correct position.
|
||||
*/
|
||||
private void truncateDestinationFileIfNecessary() {
|
||||
RandomAccessFile raf = null;
|
||||
if (!FileLocks.lock(dstfile)) {
|
||||
throw new FileLockException("Fail to lock " + dstfile);
|
||||
}
|
||||
|
||||
try {
|
||||
raf = new RandomAccessFile(dstfile, "rw");
|
||||
if (lastFullyMergedPartNumber == 0) {
|
||||
raf.setLength(0);
|
||||
} else {
|
||||
long lastByte = ServiceUtils.getLastByteInPart(s3, req, lastFullyMergedPartNumber);
|
||||
if (dstfile.length() < lastByte) {
|
||||
throw new SdkClientException(
|
||||
"File " + dstfile.getAbsolutePath() + " has been modified since last pause.");
|
||||
}
|
||||
raf.setLength(lastByte + 1);
|
||||
download.getProgress().updateProgress(lastByte + 1);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new SdkClientException("Unable to append part file to dstfile " + e.getMessage(), e);
|
||||
} finally {
|
||||
IOUtils.closeQuietly(raf, LOG);
|
||||
FileLocks.unlock(dstfile);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is called only if it is a resumed download.
|
||||
*
|
||||
* Adjust the range of the get request, and the expected (ie current) file
|
||||
* length of the destination file to append to.
|
||||
*/
|
||||
private void adjustRequest(GetObjectRequest req) {
|
||||
long[] range = req.getRange();
|
||||
long lastByte = range[1];
|
||||
long totalBytesToDownload = lastByte - this.origStartingByte + 1;
|
||||
|
||||
if (dstfile.exists()) {
|
||||
if (!FileLocks.lock(dstfile)) {
|
||||
throw new FileLockException("Fail to lock " + dstfile
|
||||
+ " for range adjustment");
|
||||
}
|
||||
try {
|
||||
expectedFileLength = dstfile.length();
|
||||
long startingByte = this.origStartingByte + expectedFileLength;
|
||||
LOG.info("Adjusting request range from " + Arrays.toString(range)
|
||||
+ " to "
|
||||
+ Arrays.toString(new long[] { startingByte, lastByte })
|
||||
+ " for file " + dstfile);
|
||||
req.setRange(startingByte, lastByte);
|
||||
totalBytesToDownload = lastByte - startingByte + 1;
|
||||
} finally {
|
||||
FileLocks.unlock(dstfile);
|
||||
}
|
||||
}
|
||||
|
||||
if (totalBytesToDownload < 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"Unable to determine the range for download operation. lastByte="
|
||||
+ lastByte + ", origStartingByte=" + origStartingByte
|
||||
+ ", expectedFileLength=" + expectedFileLength
|
||||
+ ", totalBytesToDownload=" + totalBytesToDownload);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private S3Object retryableDownloadS3ObjectToFile(File file,
|
||||
RetryableS3DownloadTask retryableS3DownloadTask) {
|
||||
boolean hasRetried = false;
|
||||
S3Object s3Object;
|
||||
for (;;) {
|
||||
final boolean appendData = resumeExistingDownload || (resumeOnRetry && hasRetried);
|
||||
if (appendData && hasRetried) {
|
||||
// Need to adjust the get range or else we risk corrupting the downloaded file
|
||||
adjustRequest(req);
|
||||
}
|
||||
s3Object = retryableS3DownloadTask.getS3ObjectStream();
|
||||
if (s3Object == null)
|
||||
return null;
|
||||
try {
|
||||
if (testing && resumeExistingDownload && !hasRetried) {
|
||||
throw new SdkClientException("testing");
|
||||
}
|
||||
ServiceUtils.downloadToFile(s3Object, file,
|
||||
retryableS3DownloadTask.needIntegrityCheck(),
|
||||
appendData, expectedFileLength);
|
||||
return s3Object;
|
||||
} catch (AmazonClientException ace) {
|
||||
if (!ace.isRetryable())
|
||||
throw ace;
|
||||
// Determine whether an immediate retry is needed according to the captured SdkClientException.
|
||||
// (There are three cases when downloadObjectToFile() throws SdkClientException:
|
||||
// 1) SocketException or SSLProtocolException when writing to disk (e.g. when user aborts the download)
|
||||
// 2) Other IOException when writing to disk
|
||||
// 3) MD5 hashes don't match
|
||||
// For 1) If SocketException is the result of the client side resetting the connection, this is retried
|
||||
// Cases 2) and 3) will always be retried
|
||||
final Throwable cause = ace.getCause();
|
||||
if ((cause instanceof SocketException && !cause.getMessage().equals("Connection reset"))
|
||||
|| (cause instanceof SSLProtocolException)) {
|
||||
throw ace;
|
||||
} else {
|
||||
if (hasRetried)
|
||||
throw ace;
|
||||
else {
|
||||
LOG.info("Retry the download of object " + s3Object.getKey() + " (bucket " + s3Object.getBucketName() + ")", ace);
|
||||
hasRetried = true;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
s3Object.getObjectContent().abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isTimeoutEnabled() {
|
||||
return timeout > 0;
|
||||
}
|
||||
|
||||
private static boolean testing;
|
||||
/**
|
||||
* Used for testing purpose only.
|
||||
*/
|
||||
static void setTesting(boolean b) {
|
||||
testing = b;
|
||||
}
|
||||
}
|
||||
@@ -1,202 +0,0 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import com.amazonaws.annotation.SdkInternalApi;
|
||||
import com.amazonaws.event.ProgressEventType;
|
||||
import com.amazonaws.event.ProgressListenerChain;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.amazonaws.services.s3.transfer.TransferProgress;
|
||||
import com.amazonaws.services.s3.transfer.exception.PauseException;
|
||||
import com.amazonaws.services.s3.transfer.internal.S3ProgressPublisher;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferManagerUtils;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
|
||||
|
||||
public class DownloadImpl extends AbstractTransfer implements Download {
|
||||
private S3Object s3Object;
|
||||
|
||||
/**
|
||||
* Information to resume if the download is paused.
|
||||
*/
|
||||
private PersistableDownload persistableDownload;
|
||||
|
||||
/**
|
||||
* The last part that has been successfully written into the downloaded file.
|
||||
*/
|
||||
private Integer lastFullyDownloadedPartNumber;
|
||||
|
||||
private final GetObjectRequest getObjectRequest;
|
||||
private final File file;
|
||||
private final ObjectMetadata objectMetadata;
|
||||
private final ProgressListenerChain progressListenerChain;
|
||||
|
||||
@Deprecated
|
||||
public DownloadImpl(String description, TransferProgress transferProgress,
|
||||
ProgressListenerChain progressListenerChain, S3Object s3Object, TransferStateChangeListener listener,
|
||||
GetObjectRequest getObjectRequest, File file) {
|
||||
this(description, transferProgress, progressListenerChain, s3Object, listener,
|
||||
getObjectRequest, file, null, false);
|
||||
}
|
||||
|
||||
public DownloadImpl(String description, TransferProgress transferProgress,
|
||||
ProgressListenerChain progressListenerChain, S3Object s3Object, TransferStateChangeListener listener,
|
||||
GetObjectRequest getObjectRequest, File file,
|
||||
ObjectMetadata objectMetadata, boolean isDownloadParallel) {
|
||||
super(description, transferProgress, progressListenerChain, listener);
|
||||
this.s3Object = s3Object;
|
||||
this.objectMetadata = objectMetadata;
|
||||
this.getObjectRequest = getObjectRequest;
|
||||
this.file = file;
|
||||
this.progressListenerChain = progressListenerChain;
|
||||
this.persistableDownload = captureDownloadState(getObjectRequest, file);
|
||||
S3ProgressPublisher.publishTransferPersistable(progressListenerChain, persistableDownload);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the ObjectMetadata for the object being downloaded.
|
||||
*
|
||||
* @return The ObjectMetadata for the object being downloaded.
|
||||
*/
|
||||
public synchronized ObjectMetadata getObjectMetadata() {
|
||||
if (s3Object != null) {
|
||||
return s3Object.getObjectMetadata();
|
||||
}
|
||||
return objectMetadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* The name of the bucket where the object is being downloaded from.
|
||||
*
|
||||
* @return The name of the bucket where the object is being downloaded from.
|
||||
*/
|
||||
public String getBucketName() {
|
||||
return getObjectRequest.getBucketName();
|
||||
}
|
||||
|
||||
/**
|
||||
* The key under which this object was stored in Amazon S3.
|
||||
*
|
||||
* @return The key under which this object was stored in Amazon S3.
|
||||
*/
|
||||
public String getKey() {
|
||||
return getObjectRequest.getKey();
|
||||
}
|
||||
|
||||
/**
|
||||
* Only for internal use.
|
||||
* For parallel downloads, Updates the persistableTransfer each time a
|
||||
* part is successfully merged into download file.
|
||||
* Then notify the listeners that new persistableTransfer is available.
|
||||
*/
|
||||
@SdkInternalApi
|
||||
public void updatePersistableTransfer(Integer lastFullyDownloadedPartNumber) {
|
||||
synchronized (this) {
|
||||
this.lastFullyDownloadedPartNumber = lastFullyDownloadedPartNumber;
|
||||
}
|
||||
|
||||
persistableDownload = captureDownloadState(getObjectRequest, file);
|
||||
S3ProgressPublisher.publishTransferPersistable(progressListenerChain, persistableDownload);
|
||||
}
|
||||
|
||||
/**
|
||||
* For parallel downloads, returns the last part number that was
|
||||
* successfully written into the download file.
|
||||
* Returns null for serial downloads.
|
||||
*/
|
||||
public synchronized Integer getLastFullyDownloadedPartNumber() {
|
||||
return lastFullyDownloadedPartNumber;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancels this download.
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized void abort() throws IOException {
|
||||
|
||||
this.monitor.getFuture().cancel(true);
|
||||
|
||||
if ( s3Object != null ) {
|
||||
s3Object.getObjectContent().abort();
|
||||
}
|
||||
setState(TransferState.Canceled);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancels this download, but skip notifying the state change listeners.
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized void abortWithoutNotifyingStateChangeListener() throws IOException {
|
||||
this.monitor.getFuture().cancel(true);
|
||||
this.state = TransferState.Canceled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the S3 object to download.
|
||||
*/
|
||||
public synchronized void setS3Object(S3Object s3Object) {
|
||||
this.s3Object = s3Object;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is also responsible for firing COMPLETED signal to the
|
||||
* listeners.
|
||||
*/
|
||||
@Override
|
||||
public void setState(TransferState state) {
|
||||
super.setState(state);
|
||||
|
||||
switch (state) {
|
||||
case Completed :
|
||||
fireProgressEvent(ProgressEventType.TRANSFER_COMPLETED_EVENT);
|
||||
break;
|
||||
case Canceled:
|
||||
fireProgressEvent(ProgressEventType.TRANSFER_CANCELED_EVENT);
|
||||
break;
|
||||
case Failed:
|
||||
fireProgressEvent(ProgressEventType.TRANSFER_FAILED_EVENT);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the captured state of the download; or null if it should not be
|
||||
* captured (for security reason).
|
||||
*/
|
||||
private PersistableDownload captureDownloadState(
|
||||
final GetObjectRequest getObjectRequest, final File file) {
|
||||
if (getObjectRequest.getSSECustomerKey() == null) {
|
||||
return new PersistableDownload(
|
||||
getObjectRequest.getBucketName(), getObjectRequest.getKey(),
|
||||
getObjectRequest.getVersionId(), getObjectRequest.getRange(),
|
||||
getObjectRequest.getResponseHeaders(), getObjectRequest.isRequesterPays(),
|
||||
file.getAbsolutePath(), getLastFullyDownloadedPartNumber(),
|
||||
getObjectMetadata().getLastModified().getTime());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
*
|
||||
* @see com.amazonaws.services.s3.transfer.Download#pause()
|
||||
*/
|
||||
@Override
|
||||
public PersistableDownload pause() throws PauseException {
|
||||
boolean forceCancel = true;
|
||||
TransferState currentState = getState();
|
||||
this.monitor.getFuture().cancel(true);
|
||||
|
||||
if (persistableDownload == null) {
|
||||
throw new PauseException(TransferManagerUtils.determinePauseStatus(
|
||||
currentState, forceCancel));
|
||||
}
|
||||
return persistableDownload;
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferMonitor;
|
||||
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
public class DownloadMonitor implements TransferMonitor {
|
||||
|
||||
private Future<?> future;
|
||||
private final DownloadImpl download;
|
||||
|
||||
public DownloadMonitor(DownloadImpl download, Future<?> future) {
|
||||
this.download = download;
|
||||
this.future = future;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized Future<?> getFuture() {
|
||||
return future;
|
||||
}
|
||||
|
||||
public synchronized void setFuture(Future<?> future) {
|
||||
this.future = future;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isDone() {
|
||||
return download.isDone();
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.util.StringUtils;
|
||||
import java.io.File;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
import com.amazonaws.SdkClientException;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
/**
|
||||
* Helper class to get a part from s3,
|
||||
* write the part data to a temporary file and
|
||||
* return the temporary file.
|
||||
*/
|
||||
public class DownloadPartCallable implements Callable<File> {
|
||||
private static final Log LOG = LogFactory.getLog(DownloadPartCallable.class);
|
||||
private static final String TEMP_FILE_MIDDLE_NAME = ".part.";
|
||||
|
||||
private final AmazonS3 s3;
|
||||
private final GetObjectRequest getPartRequest;
|
||||
private final File destinationFile;
|
||||
private final String destinationFilePath;
|
||||
|
||||
public DownloadPartCallable(AmazonS3 s3, GetObjectRequest getPartRequest, File destinationFile) {
|
||||
this.s3 = s3;
|
||||
this.getPartRequest = getPartRequest;
|
||||
this.destinationFile = destinationFile;
|
||||
this.destinationFilePath = destinationFile.getAbsolutePath();
|
||||
}
|
||||
|
||||
public File call() throws Exception {
|
||||
final File partFile = File.createTempFile(
|
||||
UUID.nameUUIDFromBytes(destinationFile.getName().getBytes(StringUtils.UTF8)).toString(),
|
||||
TEMP_FILE_MIDDLE_NAME + getPartRequest.getPartNumber().toString(),
|
||||
new File(destinationFilePath.substring(0, destinationFilePath.lastIndexOf(File.separator))));
|
||||
try {
|
||||
partFile.deleteOnExit();
|
||||
} catch (SecurityException exception) {
|
||||
LOG.warn("SecurityException denied delete access to file " + partFile.getAbsolutePath());
|
||||
}
|
||||
|
||||
if (s3.getObject(getPartRequest, partFile) == null) {
|
||||
throw new SdkClientException(
|
||||
"There is no object in S3 satisfying this request. The getObject method returned null");
|
||||
}
|
||||
return partFile;
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3Encryption;
|
||||
import com.amazonaws.services.s3.internal.ServiceUtils;
|
||||
import com.amazonaws.services.s3.internal.SkipMd5CheckStrategy;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
|
||||
final class DownloadTaskImpl implements
|
||||
ServiceUtils.RetryableS3DownloadTask
|
||||
{
|
||||
private final AmazonS3 s3;
|
||||
private final DownloadImpl download;
|
||||
private final GetObjectRequest getObjectRequest;
|
||||
private final SkipMd5CheckStrategy skipMd5CheckStrategy = SkipMd5CheckStrategy.INSTANCE;
|
||||
|
||||
DownloadTaskImpl(AmazonS3 s3, DownloadImpl download,
|
||||
GetObjectRequest getObjectRequest) {
|
||||
this.s3 = s3;
|
||||
this.download = download;
|
||||
this.getObjectRequest = getObjectRequest;
|
||||
}
|
||||
|
||||
@Override
|
||||
public S3Object getS3ObjectStream() {
|
||||
S3Object s3Object = s3.getObject(getObjectRequest);
|
||||
download.setS3Object(s3Object);
|
||||
return s3Object;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean needIntegrityCheck() {
|
||||
// Don't perform the integrity check if the checksum won't matchup.
|
||||
return !(s3 instanceof AmazonS3Encryption) && !skipMd5CheckStrategy.skipClientSideValidationPerRequest(getObjectRequest);
|
||||
}
|
||||
}
|
||||
@@ -1,159 +0,0 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.services.s3.model.ResponseHeaderOverrides;
|
||||
import com.amazonaws.services.s3.transfer.PersistableTransfer;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
/**
|
||||
* An opaque token that holds some private state and can be used to resume a
|
||||
* paused download operation.
|
||||
*/
|
||||
public final class PersistableDownload extends PersistableTransfer {
|
||||
|
||||
static final String TYPE = "download";
|
||||
|
||||
@JsonProperty
|
||||
private final String pauseType = TYPE;
|
||||
|
||||
/** The bucket name in Amazon S3 from where the object has to be downloaded. */
|
||||
@JsonProperty
|
||||
private final String bucketName;
|
||||
|
||||
/** The name of the object in Amazon S3 that has to be downloaded. */
|
||||
@JsonProperty
|
||||
private final String key;
|
||||
|
||||
/** The version id of the object in Amazon S3 to download. */
|
||||
@JsonProperty
|
||||
private final String versionId;
|
||||
|
||||
/** Optional member indicating the byte range of data to retrieve */
|
||||
@JsonProperty
|
||||
private final long[] range;
|
||||
|
||||
/**
|
||||
* Optional field that overrides headers on the response.
|
||||
*/
|
||||
@JsonProperty
|
||||
private final ResponseHeaderOverrides responseHeaders;
|
||||
|
||||
/**
|
||||
* If enabled, the requester is charged for downloading the data from
|
||||
* Requester Pays Buckets.
|
||||
*/
|
||||
@JsonProperty
|
||||
private final boolean isRequesterPays;
|
||||
|
||||
/**
|
||||
* File where the downloaded data is written.
|
||||
*/
|
||||
@JsonProperty
|
||||
private final String file;
|
||||
|
||||
/**
|
||||
* The last part that has been successfully written into the downloaded file.
|
||||
*/
|
||||
@JsonProperty
|
||||
private final Integer lastFullyDownloadedPartNumber;
|
||||
|
||||
/**
|
||||
* Last Modified/created time on Amazon S3 for this object.
|
||||
*/
|
||||
@JsonProperty
|
||||
private final long lastModifiedTime;
|
||||
|
||||
|
||||
public PersistableDownload() {
|
||||
this(null, null, null, null, null, false, null, null, 0L);
|
||||
}
|
||||
|
||||
public PersistableDownload(
|
||||
@JsonProperty(value = "bucketName") String bucketName,
|
||||
@JsonProperty(value = "key") String key,
|
||||
@JsonProperty(value = "versionId") String versionId,
|
||||
@JsonProperty(value = "range") long[] range,
|
||||
@JsonProperty(value = "responseHeaders") ResponseHeaderOverrides responseHeaders,
|
||||
@JsonProperty(value = "isRequesterPays") boolean isRequesterPays,
|
||||
@JsonProperty(value = "file") String file,
|
||||
@JsonProperty(value = "lastFullyDownloadedPartNumber") Integer lastFullyDownloadedPartNumber,
|
||||
@JsonProperty(value = "lastModifiedTime") long lastModifiedTime) {
|
||||
this.bucketName = bucketName;
|
||||
this.key = key;
|
||||
this.versionId = versionId;
|
||||
this.range = range == null ? null : range.clone();
|
||||
this.responseHeaders = responseHeaders;
|
||||
this.isRequesterPays = isRequesterPays;
|
||||
this.file = file;
|
||||
this.lastFullyDownloadedPartNumber = lastFullyDownloadedPartNumber;
|
||||
this.lastModifiedTime = lastModifiedTime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the name of the bucket.
|
||||
*/
|
||||
String getBucketName() {
|
||||
return bucketName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the name of the object.
|
||||
*/
|
||||
String getKey() {
|
||||
return key;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the version id of the object.
|
||||
*/
|
||||
String getVersionId() {
|
||||
return versionId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the byte range of the object to download.
|
||||
*/
|
||||
long[] getRange() {
|
||||
return range == null ? null : range.clone();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the optional response headers.
|
||||
*/
|
||||
ResponseHeaderOverrides getResponseHeaders() {
|
||||
return responseHeaders;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if RequesterPays is enabled on the Amazon S3 bucket else
|
||||
* false.
|
||||
*/
|
||||
boolean isRequesterPays() {
|
||||
return isRequesterPays;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the file where the object is to be downloaded.
|
||||
*/
|
||||
String getFile() {
|
||||
return file;
|
||||
}
|
||||
|
||||
String getPauseType() {
|
||||
return pauseType;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the last part number that was successfully written into the downloaded file.
|
||||
*/
|
||||
Integer getLastFullyDownloadedPartNumber() {
|
||||
return lastFullyDownloadedPartNumber;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the last modified/created time of the object represented by
|
||||
* the bucketName and key.
|
||||
*/
|
||||
Long getlastModifiedTime() {
|
||||
return lastModifiedTime;
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.event.ProgressEvent;
|
||||
import com.amazonaws.event.ProgressEventFilter;
|
||||
import com.amazonaws.event.ProgressEventType;
|
||||
|
||||
final class TransferCompletionFilter implements ProgressEventFilter {
|
||||
@Override
|
||||
public ProgressEvent filter(ProgressEvent progressEvent) {
|
||||
// Block COMPLETE events from the low-level GetObject operation,
|
||||
// but we still want to keep the BytesTransferred
|
||||
return progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT
|
||||
? null // discard this event
|
||||
: progressEvent
|
||||
;
|
||||
}
|
||||
}
|
||||
@@ -1,233 +0,0 @@
|
||||
package org.talend.aws;
|
||||
|
||||
import com.amazonaws.AmazonClientException;
|
||||
import com.amazonaws.AmazonWebServiceRequest;
|
||||
import com.amazonaws.event.ProgressListenerChain;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.internal.FileLocks;
|
||||
import com.amazonaws.services.s3.internal.RequestCopyUtils;
|
||||
import com.amazonaws.services.s3.internal.ServiceUtils;
|
||||
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.transfer.TransferManagerConfiguration;
|
||||
import com.amazonaws.services.s3.transfer.TransferProgress;
|
||||
import com.amazonaws.services.s3.transfer.exception.FileLockException;
|
||||
import com.amazonaws.services.s3.transfer.internal.S3ProgressListener;
|
||||
import com.amazonaws.services.s3.transfer.internal.S3ProgressListenerChain;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferManagerUtils;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
|
||||
import com.amazonaws.services.s3.transfer.internal.TransferProgressUpdatingListener;
|
||||
import com.amazonaws.util.VersionInfoUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Date;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
public class TransferManager {
|
||||
|
||||
private static final Log log = LogFactory.getLog(TransferManager.class);
|
||||
|
||||
private final AmazonS3 s3;
|
||||
private final ExecutorService executorService;
|
||||
private final TransferManagerConfiguration configuration;
|
||||
private final boolean shutDownThreadPools;
|
||||
|
||||
public TransferManager(AmazonS3 s3) {
|
||||
this.s3 = s3;
|
||||
this.executorService = TransferManagerUtils.createDefaultExecutorService();
|
||||
this.configuration = resolveConfiguration();
|
||||
this.shutDownThreadPools = true;
|
||||
}
|
||||
|
||||
private TransferManagerConfiguration resolveConfiguration() {
|
||||
TransferManagerConfiguration configuration = new TransferManagerConfiguration();
|
||||
configuration.setDisableParallelDownloads(false);
|
||||
return configuration;
|
||||
}
|
||||
|
||||
public Download download(GetObjectRequest getObjectRequest, File file, S3ProgressListener progressListener,
|
||||
long timeoutMillis, boolean resumeOnRetry) {
|
||||
return doDownload(getObjectRequest, file, null, progressListener, ServiceUtils.OVERWRITE_MODE, timeoutMillis, null, 0L,
|
||||
resumeOnRetry);
|
||||
}
|
||||
|
||||
private Download doDownload(final GetObjectRequest getObjectRequest,
|
||||
final File file, final TransferStateChangeListener stateListener,
|
||||
final S3ProgressListener s3progressListener,
|
||||
final boolean resumeExistingDownload,
|
||||
final long timeoutMillis,
|
||||
final Integer lastFullyDownloadedPart,
|
||||
final long lastModifiedTimeRecordedDuringPause,
|
||||
final boolean resumeOnRetry)
|
||||
{
|
||||
assertParameterNotNull(getObjectRequest,
|
||||
"A valid GetObjectRequest must be provided to initiate download");
|
||||
assertParameterNotNull(file,
|
||||
"A valid file must be provided to download into");
|
||||
|
||||
appendSingleObjectUserAgent(getObjectRequest);
|
||||
String description = "Downloading from " + getObjectRequest.getBucketName() + "/" + getObjectRequest.getKey();
|
||||
|
||||
TransferProgress transferProgress = new TransferProgress();
|
||||
// S3 progress listener to capture the persistable transfer when available
|
||||
S3ProgressListenerChain listenerChain = new S3ProgressListenerChain(
|
||||
// The listener for updating transfer progress
|
||||
new TransferProgressUpdatingListener(transferProgress),
|
||||
getObjectRequest.getGeneralProgressListener(),
|
||||
s3progressListener); // Listeners included in the original request
|
||||
// The listener chain used by the low-level GetObject request.
|
||||
// This listener chain ignores any COMPLETE event, so that we could
|
||||
// delay firing the signal until the high-level download fully finishes.
|
||||
getObjectRequest
|
||||
.setGeneralProgressListener(new ProgressListenerChain(new TransferCompletionFilter(), listenerChain));
|
||||
|
||||
GetObjectMetadataRequest getObjectMetadataRequest = RequestCopyUtils.createGetObjectMetadataRequestFrom(getObjectRequest);
|
||||
final ObjectMetadata objectMetadata = s3.getObjectMetadata(getObjectMetadataRequest);
|
||||
|
||||
// Used to check if the object is modified between pause and resume
|
||||
long lastModifiedTime = objectMetadata.getLastModified().getTime();
|
||||
|
||||
long startingByte = 0;
|
||||
long lastByte;
|
||||
|
||||
long[] range = getObjectRequest.getRange();
|
||||
if (range != null && range.length == 2) {
|
||||
startingByte = range[0];
|
||||
lastByte = range[1];
|
||||
} else {
|
||||
lastByte = objectMetadata.getContentLength() - 1;
|
||||
}
|
||||
|
||||
final long origStartingByte = startingByte;
|
||||
final boolean isDownloadParallel = !configuration.isDisableParallelDownloads()
|
||||
&& TransferManagerUtils.isDownloadParallelizable(s3, getObjectRequest, ServiceUtils.getPartCount(getObjectRequest, s3));
|
||||
|
||||
// We still pass the unfiltered listener chain into DownloadImpl
|
||||
final DownloadImpl download = new DownloadImpl(description, transferProgress, listenerChain, null,
|
||||
stateListener, getObjectRequest, file, objectMetadata, isDownloadParallel);
|
||||
|
||||
long totalBytesToDownload = lastByte - startingByte + 1;
|
||||
transferProgress.setTotalBytesToTransfer(totalBytesToDownload);
|
||||
|
||||
// Range information is needed for auto retry of downloads so a retry
|
||||
// request can start at the last downloaded location in the range.
|
||||
//
|
||||
// For obvious reasons, setting a Range header only makes sense if the
|
||||
// object actually has content because it's inclusive, otherwise S3
|
||||
// responds with 4xx
|
||||
//
|
||||
// In addition, we only set the range if the download was *NOT*
|
||||
// determined to be parallelizable above. One of the conditions for
|
||||
// parallel downloads is that getRange() returns null so preserve that.
|
||||
if (totalBytesToDownload > 0 && !isDownloadParallel) {
|
||||
getObjectRequest.withRange(startingByte, lastByte);
|
||||
}
|
||||
|
||||
long fileLength = -1;
|
||||
|
||||
if (resumeExistingDownload) {
|
||||
if (isS3ObjectModifiedSincePause(lastModifiedTime, lastModifiedTimeRecordedDuringPause)) {
|
||||
throw new AmazonClientException("The requested object in bucket " + getObjectRequest.getBucketName()
|
||||
+ " with key " + getObjectRequest.getKey() + " is modified on Amazon S3 since the last pause.");
|
||||
}
|
||||
// There's still a chance the object is modified while the request
|
||||
// is in flight. Set this header so S3 fails the request if this happens.
|
||||
getObjectRequest.setUnmodifiedSinceConstraint(new Date(lastModifiedTime));
|
||||
|
||||
if (!isDownloadParallel) {
|
||||
if (!FileLocks.lock(file)) {
|
||||
throw new FileLockException("Fail to lock " + file + " for resume download");
|
||||
}
|
||||
try {
|
||||
if (file.exists()) {
|
||||
fileLength = file.length();
|
||||
startingByte = startingByte + fileLength;
|
||||
getObjectRequest.setRange(startingByte, lastByte);
|
||||
transferProgress.updateProgress(Math.min(fileLength, totalBytesToDownload));
|
||||
totalBytesToDownload = lastByte - startingByte + 1;
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("Resume download: totalBytesToDownload=" + totalBytesToDownload
|
||||
+ ", origStartingByte=" + origStartingByte + ", startingByte=" + startingByte
|
||||
+ ", lastByte=" + lastByte + ", numberOfBytesRead=" + fileLength + ", file: "
|
||||
+ file);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
FileLocks.unlock(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (totalBytesToDownload < 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"Unable to determine the range for download operation.");
|
||||
}
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(1);
|
||||
Future<?> future = executorService.submit(
|
||||
new DownloadCallable(s3, latch,
|
||||
getObjectRequest, resumeExistingDownload,
|
||||
download, file, origStartingByte, fileLength, timeoutMillis, timedThreadPool,
|
||||
executorService, lastFullyDownloadedPart, isDownloadParallel, resumeOnRetry));
|
||||
download.setMonitor(new DownloadMonitor(download, future));
|
||||
latch.countDown();
|
||||
return download;
|
||||
}
|
||||
|
||||
public void shutdownNow(boolean shutDownS3Client) {
|
||||
if (shutDownThreadPools) {
|
||||
executorService.shutdownNow();
|
||||
timedThreadPool.shutdownNow();
|
||||
}
|
||||
|
||||
if (shutDownS3Client) {
|
||||
s3.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
private void assertParameterNotNull(Object parameterValue, String errorMessage) {
|
||||
if (parameterValue == null) throw new IllegalArgumentException(errorMessage);
|
||||
}
|
||||
|
||||
public static <X extends AmazonWebServiceRequest> X appendSingleObjectUserAgent(X request) {
|
||||
request.getRequestClientOptions().appendUserAgent(USER_AGENT);
|
||||
return request;
|
||||
}
|
||||
|
||||
private static final String USER_AGENT = TransferManager.class.getName() + "/" + VersionInfoUtils.getVersion();
|
||||
|
||||
private boolean isS3ObjectModifiedSincePause(final long lastModifiedTimeRecordedDuringResume,
|
||||
long lastModifiedTimeRecordedDuringPause) {
|
||||
return lastModifiedTimeRecordedDuringResume != lastModifiedTimeRecordedDuringPause;
|
||||
}
|
||||
|
||||
private final ScheduledExecutorService timedThreadPool = new ScheduledThreadPoolExecutor(1, daemonThreadFactory);
|
||||
|
||||
private static final ThreadFactory daemonThreadFactory = new ThreadFactory() {
|
||||
final AtomicInteger threadCount = new AtomicInteger( 0 );
|
||||
public Thread newThread(Runnable r) {
|
||||
int threadNumber = threadCount.incrementAndGet();
|
||||
Thread thread = new Thread(r);
|
||||
thread.setDaemon(true);
|
||||
thread.setName("S3TransferManagerTimedThread-" + threadNumber);
|
||||
return thread;
|
||||
}
|
||||
};
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
shutdownThreadPools();
|
||||
}
|
||||
|
||||
private void shutdownThreadPools() {
|
||||
if (shutDownThreadPools) {
|
||||
executorService.shutdown();
|
||||
timedThreadPool.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -4,7 +4,7 @@
|
||||
<groupId>org.talend</groupId>
|
||||
<artifactId>talend-httputil</artifactId>
|
||||
<name>talend-httputil</name>
|
||||
<version>1.0.6</version>
|
||||
<version>1.0.5</version>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
@@ -20,7 +20,7 @@
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>2.11.4</version>
|
||||
<version>2.10.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
@@ -6,11 +6,11 @@
|
||||
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>job-audit</artifactId>
|
||||
<version>1.4</version>
|
||||
<version>1.2</version>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<daikon.audit.version>1.16.1</daikon.audit.version>
|
||||
<daikon.audit.version>1.16.0</daikon.audit.version>
|
||||
</properties>
|
||||
|
||||
<repositories>
|
||||
|
||||
@@ -24,11 +24,5 @@ public interface JobAuditLogger extends EventAuditLogger {
|
||||
|
||||
@AuditEvent(category = "flowexecution", message = "connection : {connection_name}, row : {rows}, cost : {duration}", level = LogLevel.INFO)
|
||||
void flowExecution(Context context);
|
||||
|
||||
@AuditEvent(category = "componentparameters", message = "Component {connector_id} parameters", level = LogLevel.INFO)
|
||||
void componentParameters(Context context);
|
||||
|
||||
@AuditEvent(category = "schema", message = "{connection_name} : {schema} from {source_id} to {target_id}", level = LogLevel.INFO)
|
||||
void schema(Context context);
|
||||
|
||||
}
|
||||
|
||||
@@ -15,11 +15,6 @@ public class JobContextBuilder {
|
||||
return new JobContextBuilder(ContextBuilder.create());
|
||||
}
|
||||
|
||||
public JobContextBuilder custom(String key, String value) {
|
||||
builder.with(key, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder jobName(String job_name) {
|
||||
builder.with("job_name", job_name);
|
||||
return this;
|
||||
@@ -166,14 +161,4 @@ public class JobContextBuilder {
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder connectorParameters(String connector_parameters) {
|
||||
builder.with("connector_parameters", connector_parameters);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JobContextBuilder schema(String schema) {
|
||||
builder.with("schema", schema);
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>talendExcel</artifactId>
|
||||
<version>1.9-20201223</version>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talendExcel-1.5-20190731</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>talendExcel</name>
|
||||
@@ -43,30 +43,48 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi</artifactId>
|
||||
<version>4.1.2</version>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-scratchpad</artifactId>
|
||||
<version>4.1.2</version>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml</artifactId>
|
||||
<version>4.1.2</version>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml-schemas -->
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml-schemas</artifactId>
|
||||
<version>4.1.2</version>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.geronimo.specs/geronimo-stax-api_1.0_spec -->
|
||||
<dependency>
|
||||
<groupId>org.apache.geronimo.specs</groupId>
|
||||
<artifactId>geronimo-stax-api_1.0_spec</artifactId>
|
||||
<version>1.0</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/dom4j/dom4j -->
|
||||
<dependency>
|
||||
<groupId>dom4j</groupId>
|
||||
<artifactId>dom4j</artifactId>
|
||||
<version>1.6.1</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.xmlbeans/xmlbeans -->
|
||||
<dependency>
|
||||
<groupId>org.apache.xmlbeans</groupId>
|
||||
<artifactId>xmlbeans</artifactId>
|
||||
<version>3.1.0</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
|
||||
@@ -6,13 +6,11 @@ import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.security.GeneralSecurityException;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.poi.poifs.crypt.Decryptor;
|
||||
import org.apache.poi.poifs.crypt.EncryptionInfo;
|
||||
import org.apache.poi.poifs.crypt.EncryptionMode;
|
||||
import org.apache.poi.poifs.crypt.Encryptor;
|
||||
@@ -35,12 +33,12 @@ public class ExcelTool {
|
||||
|
||||
private Workbook wb = null;
|
||||
|
||||
private Workbook preWb = null;
|
||||
|
||||
private String sheetName = null;
|
||||
|
||||
private Sheet sheet = null;
|
||||
|
||||
private Workbook preWb = null;
|
||||
|
||||
private Sheet preSheet = null;
|
||||
|
||||
private Row curRow = null;
|
||||
@@ -80,8 +78,6 @@ public class ExcelTool {
|
||||
private boolean isTrackAllColumns = false;
|
||||
|
||||
private String password = null;
|
||||
|
||||
private Map<CellStyle, CellStyle> existedOriginToClone;
|
||||
|
||||
public ExcelTool() {
|
||||
cellStylesMapping = new HashMap<>();
|
||||
@@ -162,9 +158,9 @@ public class ExcelTool {
|
||||
}
|
||||
}
|
||||
|
||||
private void appendActionForFile(String fileName) throws Exception {
|
||||
private void appendActionForFile(String fileName) throws IOException {
|
||||
InputStream inp = new FileInputStream(fileName);
|
||||
wb = WorkbookFactory.create(inp, password);
|
||||
wb = WorkbookFactory.create(inp);
|
||||
sheet = wb.getSheet(sheetName);
|
||||
if (sheet != null) {
|
||||
if (appendSheet) {
|
||||
@@ -190,7 +186,7 @@ public class ExcelTool {
|
||||
|
||||
private void initPreXlsx(String fileName) throws Exception {
|
||||
InputStream preIns = new FileInputStream(fileName);
|
||||
preWb = WorkbookFactory.create(preIns, password);
|
||||
preWb = WorkbookFactory.create(preIns);
|
||||
preSheet = preWb.getSheet(sheetName);
|
||||
}
|
||||
|
||||
@@ -203,7 +199,11 @@ public class ExcelTool {
|
||||
|
||||
public void addRow() {
|
||||
if (isAbsY && keepCellFormat) {
|
||||
preRow = (preSheet != null) ? preSheet.getRow(curY) : null;
|
||||
if (preSheet != null) {
|
||||
preRow = preSheet.getRow(curY);
|
||||
} else {
|
||||
preRow = null;
|
||||
}
|
||||
}
|
||||
curRow = sheet.getRow(curY);
|
||||
if (curRow == null) {
|
||||
@@ -221,7 +221,11 @@ public class ExcelTool {
|
||||
|
||||
private void addCell() {
|
||||
if (isAbsY && keepCellFormat) {
|
||||
preCell = (preRow != null) ? preRow.getCell(startX + xOffset) : null;
|
||||
if (preRow != null) {
|
||||
preCell = preRow.getCell(startX + xOffset);
|
||||
} else {
|
||||
preCell = null;
|
||||
}
|
||||
}
|
||||
curCell = curRow.createCell(startX + xOffset);
|
||||
xOffset++;
|
||||
@@ -240,8 +244,10 @@ public class ExcelTool {
|
||||
cellStylesMapping.put("normal", style);
|
||||
return style;
|
||||
}
|
||||
} else {
|
||||
return preCellStyle;
|
||||
}
|
||||
return preCellStyle;
|
||||
|
||||
}
|
||||
|
||||
private CellStyle getDateCellStyle(String pattern) {
|
||||
@@ -260,15 +266,13 @@ public class ExcelTool {
|
||||
cellStylesMapping.put(pattern, style);
|
||||
return style;
|
||||
}
|
||||
} else {
|
||||
return preCellStyle;
|
||||
}
|
||||
return preCellStyle;
|
||||
}
|
||||
|
||||
private CellStyle getPreCellStyle() {
|
||||
if (preSheet != null && isAbsY && keepCellFormat) {
|
||||
if(existedOriginToClone==null) {
|
||||
existedOriginToClone = new HashMap<>();
|
||||
}
|
||||
CellStyle preCellStyle;
|
||||
if (preCell == null) {
|
||||
preCellStyle = preSheet.getColumnStyle(curCell.getColumnIndex());
|
||||
@@ -276,17 +280,14 @@ public class ExcelTool {
|
||||
preCellStyle = preCell.getCellStyle();
|
||||
}
|
||||
|
||||
CellStyle targetCellStyle = existedOriginToClone.get(preCellStyle);
|
||||
if(targetCellStyle==null) {
|
||||
targetCellStyle = wb.createCellStyle();
|
||||
targetCellStyle.cloneStyleFrom(preCellStyle);
|
||||
existedOriginToClone.put(preCellStyle, targetCellStyle);
|
||||
}
|
||||
CellStyle targetCellStyle = wb.createCellStyle();
|
||||
targetCellStyle.cloneStyleFrom(preCellStyle);
|
||||
|
||||
return targetCellStyle;
|
||||
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public void addCellValue(boolean booleanValue) {
|
||||
@@ -340,13 +341,7 @@ public class ExcelTool {
|
||||
try {
|
||||
wb.write(outputStream);
|
||||
wb.close();
|
||||
if(preWb != null){
|
||||
preWb.close();
|
||||
}
|
||||
} finally {
|
||||
if(existedOriginToClone!=null) {
|
||||
existedOriginToClone = null;
|
||||
}
|
||||
if (outputStream != null) {
|
||||
outputStream.close();
|
||||
}
|
||||
@@ -364,11 +359,13 @@ public class ExcelTool {
|
||||
if (appendWorkbook && appendSheet && recalculateFormula) {
|
||||
evaluateFormulaCell();
|
||||
}
|
||||
try (FileOutputStream fileOutput = new FileOutputStream(fileName);
|
||||
POIFSFileSystem fs = new POIFSFileSystem()) {
|
||||
FileOutputStream fileOutput = new FileOutputStream(fileName);
|
||||
POIFSFileSystem fs = null;
|
||||
try {
|
||||
if (password == null) {
|
||||
wb.write(fileOutput);
|
||||
} else {
|
||||
fs = new POIFSFileSystem();
|
||||
Encryptor encryptor = new EncryptionInfo(EncryptionMode.agile).getEncryptor();
|
||||
encryptor.confirmPassword(password);
|
||||
OutputStream encryptedDataStream = encryptor.getDataStream(fs);
|
||||
@@ -377,12 +374,10 @@ public class ExcelTool {
|
||||
fs.writeFilesystem(fileOutput);
|
||||
}
|
||||
} finally {
|
||||
if(existedOriginToClone!=null) {
|
||||
existedOriginToClone = null;
|
||||
}
|
||||
wb.close();
|
||||
if(preWb != null){
|
||||
preWb.close();
|
||||
fileOutput.close();
|
||||
if (fs != null) {
|
||||
fs.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -393,8 +388,8 @@ public class ExcelTool {
|
||||
sheet = wb.getSheetAt(sheetNum);
|
||||
for (Row r : sheet) {
|
||||
for (Cell c : r) {
|
||||
if (c.getCellType() == CellType.FORMULA) {
|
||||
evaluator.evaluateFormulaCell(c);
|
||||
if (c.getCellTypeEnum() == CellType.FORMULA) {
|
||||
evaluator.evaluateFormulaCellEnum(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,61 +1,61 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>talendMsgMailUtil</artifactId>
|
||||
<version>1.2-20200923</version>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talendMsgMailUtil-1.1-20191012</artifactId>
|
||||
<name>talendMsgMailUtil</name>
|
||||
<version>6.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<java.source.version>1.8</java.source.version>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
</snapshotRepository>
|
||||
<repository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<java.source.version>1.5</java.source.version>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
</snapshotRepository>
|
||||
<repository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.25</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.25</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi</artifactId>
|
||||
<version>4.1.2</version>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-scratchpad</artifactId>
|
||||
<version>4.1.2</version>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17</version>
|
||||
<version>1.2.13</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
@@ -65,19 +65,19 @@
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>${java.source.version}</source>
|
||||
<target>${java.source.version}</target>
|
||||
<showDeprecation>true</showDeprecation>
|
||||
<showWarnings>true</showWarnings>
|
||||
<compilerArgument>-XDignore.symbol.file</compilerArgument>
|
||||
<fork>true</fork>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>${java.source.version}</source>
|
||||
<target>${java.source.version}</target>
|
||||
<showDeprecation>true</showDeprecation>
|
||||
<showWarnings>true</showWarnings>
|
||||
<compilerArgument>-XDignore.symbol.file</compilerArgument>
|
||||
<fork>true</fork>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@@ -7,6 +7,10 @@ import java.io.OutputStream;
|
||||
|
||||
import org.apache.poi.hsmf.MAPIMessage;
|
||||
import org.apache.poi.hsmf.datatypes.AttachmentChunks;
|
||||
import org.apache.poi.hsmf.datatypes.Chunk;
|
||||
import org.apache.poi.hsmf.datatypes.MAPIProperty;
|
||||
import org.apache.poi.hsmf.datatypes.StringChunk;
|
||||
import org.apache.poi.hsmf.datatypes.Types;
|
||||
import org.apache.poi.hsmf.exceptions.ChunkNotFoundException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -32,12 +36,12 @@ public class MsgMailUtil {
|
||||
|
||||
public MsgMailUtil(String fileName, String outAttachmentPath)
|
||||
throws IOException {
|
||||
this.msg = new MAPIMessage(fileName);
|
||||
msg = new MAPIMessage(fileName);
|
||||
this.outAttachmentPath = outAttachmentPath;
|
||||
}
|
||||
|
||||
public void activeLog(String loggerName, String position) {
|
||||
this.log = LoggerFactory.getLogger(loggerName);
|
||||
public void activeLog(String logger_name, String position) {
|
||||
this.log = LoggerFactory.getLogger(logger_name);
|
||||
this.position = position;
|
||||
}
|
||||
|
||||
@@ -66,14 +70,20 @@ public class MsgMailUtil {
|
||||
}
|
||||
|
||||
File attachedFile = new File(dir, fileName);
|
||||
try(OutputStream fileOut = new FileOutputStream(attachedFile)) {
|
||||
OutputStream fileOut = null;
|
||||
try {
|
||||
processLog(Level.INFO, "Exporting attachment file :" + fileName);
|
||||
processLog(Level.INFO,
|
||||
"File location:" + attachedFile.getAbsolutePath());
|
||||
|
||||
fileOut = new FileOutputStream(attachedFile);
|
||||
fileOut.write(attachment.getEmbeddedAttachmentObject());
|
||||
|
||||
processLog(Level.INFO, "Export successfully");
|
||||
} finally {
|
||||
if (fileOut != null) {
|
||||
fileOut.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.components.lib</groupId>
|
||||
<artifactId>talend_file_enhanced</artifactId>
|
||||
<name>talend_file_enhanced</name>
|
||||
<version>1.0</version>
|
||||
|
||||
<properties>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.0</version>
|
||||
<type>jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
|
||||
<snapshots><enabled>true</enabled></snapshots>
|
||||
<releases><enabled>false</enabled></releases>
|
||||
</snapshotRepository>
|
||||
<repository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
<snapshots><enabled>false</enabled></snapshots>
|
||||
<releases><enabled>true</enabled></releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
<encoding>UTF-8</encoding>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/java</directory>
|
||||
</resource>
|
||||
</resources>
|
||||
</build>
|
||||
</project>
|
||||
@@ -0,0 +1,89 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project name="org.talend.designer.components.libs" default="process" basedir=".">
|
||||
|
||||
<property name="jar.plugin.home" value="../../../org.talend.libraries.custom/lib" />
|
||||
|
||||
<!-- #################################################### -->
|
||||
|
||||
<!-- modification 1: config -->
|
||||
<property name="jar.name" value="talend_file_enhanced_20070724.jar" />
|
||||
<property name="author.name" value="wyang" />
|
||||
|
||||
<!-- modification 2: compile classpath -->
|
||||
<path id="compile.classpath">
|
||||
</path>
|
||||
|
||||
<!-- #################################################### -->
|
||||
|
||||
|
||||
<!-- sourcecode and final jar path -->
|
||||
<property name="source.home" value="." />
|
||||
<property name="jar.home" value="${jar.plugin.home}/${jar.name}" />
|
||||
|
||||
<!-- temp dir for clasee files -->
|
||||
<property name="build.dir" value="../../build" />
|
||||
|
||||
<!-- compile option -->
|
||||
<property name="compile.debug" value="true" />
|
||||
<property name="compile.deprecation" value="false" />
|
||||
<property name="compile.optimize" value="true" />
|
||||
|
||||
<property name="ant.build.javac.source" value="1.6"/>
|
||||
<property name="ant.build.javac.target" value="1.6"/>
|
||||
|
||||
<target name="process" description="prepare a temp dir">
|
||||
<antcall target="prepare" />
|
||||
<antcall target="compile" />
|
||||
<antcall target="clean" />
|
||||
</target>
|
||||
|
||||
<target name="prepare" description="prepare a temp dir">
|
||||
<delete dir="${build.dir}" />
|
||||
<mkdir dir="${build.dir}" />
|
||||
<mkdir dir="${build.dir}/classes" />
|
||||
</target>
|
||||
|
||||
<target name="compile" description="Compile Java sources">
|
||||
|
||||
<!-- compile -->
|
||||
<javac srcdir="${source.home}" destdir="${build.dir}/classes" debug="${compile.debug}" deprecation="${compile.deprecation}" optimize="${compile.optimize}">
|
||||
<classpath refid="compile.classpath" />
|
||||
</javac>
|
||||
|
||||
<!-- include source code -->
|
||||
<copy todir="${build.dir}/classes">
|
||||
<fileset dir="${source.home}">
|
||||
<exclude name="build.xml" />
|
||||
</fileset>
|
||||
</copy>
|
||||
|
||||
<!-- make jar -->
|
||||
<tstamp>
|
||||
<format property="date" pattern="yyyy-MM-dd HH:mm:ss" />
|
||||
</tstamp>
|
||||
|
||||
<jar destfile="${build.dir}/${jar.name}" basedir="${build.dir}/classes">
|
||||
<manifest>
|
||||
<!-- who -->
|
||||
<attribute name="Built-By" value="${author.name}" />
|
||||
<!-- when -->
|
||||
<attribute name="Built-Date" value="${date}"/>
|
||||
<!-- JDK version -->
|
||||
<attribute name="Created-By" value="${java.version} (${java.vendor})" />
|
||||
<!-- Information about the program itself -->
|
||||
<attribute name="Implementation-Vendor" value="Talend SA" />
|
||||
<attribute name="Implementation-Title" value="${jar.name}" />
|
||||
<attribute name="Implementation-Version" value="1.0" />
|
||||
</manifest>
|
||||
</jar>
|
||||
|
||||
<!-- move jar -->
|
||||
<move file="${build.dir}/${jar.name}" tofile="${jar.home}" />
|
||||
|
||||
</target>
|
||||
|
||||
<target name="clean" description="clean the temp dir">
|
||||
<delete dir="${build.dir}" />
|
||||
<mkdir dir="${build.dir}" />
|
||||
</target>
|
||||
</project>
|
||||
@@ -561,17 +561,7 @@ public class TOSDelimitedReader {
|
||||
System.out.println(streamBuffer);
|
||||
}
|
||||
|
||||
setStreamEnd(readCount, maxReadLength);
|
||||
|
||||
if (readCount > -1) {
|
||||
count += readCount;
|
||||
lastIndexToRead = count - maxLimit;
|
||||
}
|
||||
}
|
||||
|
||||
private void setStreamEnd(int readCount, int maxReadLength) throws IOException {
|
||||
/* @see bug:http://talendforge.org/bugs/view.php?id=4554 */
|
||||
//https://jira.talendforge.org/browse/TDI-44745
|
||||
if (readCount < maxReadLength) {
|
||||
if (readCount == -1) {
|
||||
streamEndMeet = true;
|
||||
@@ -585,6 +575,11 @@ public class TOSDelimitedReader {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (readCount > -1) {
|
||||
count += readCount;
|
||||
lastIndexToRead = count - maxLimit;
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------1------------------- */
|
||||
@@ -657,7 +652,7 @@ public class TOSDelimitedReader {
|
||||
currentPosition = 0;
|
||||
columnStart = 0;
|
||||
lastIndexToRead = count - maxLimit;
|
||||
setStreamEnd(count, buffer.length);
|
||||
streamEndMeet = (count < buffer.length);
|
||||
}
|
||||
|
||||
public boolean isStartFieldDelimited() {
|
||||
@@ -37,7 +37,7 @@
|
||||
<dependency>
|
||||
<groupId>org.dom4j</groupId>
|
||||
<artifactId>dom4j</artifactId>
|
||||
<version>2.1.3</version>
|
||||
<version>2.1.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
@@ -2,21 +2,9 @@
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talendzip</artifactId>
|
||||
<version>1.1-20201120</version>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<source>8</source>
|
||||
<target>8</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<version>1.0-20190917</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
|
||||
@@ -2,7 +2,6 @@ package com.talend.compress.zip;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import net.lingala.zip4j.core.ZipFile;
|
||||
import net.lingala.zip4j.model.FileHeader;
|
||||
@@ -45,8 +44,6 @@ public class Unzip {
|
||||
this.useZip4jDecryption = useZip4jDecryption;
|
||||
}
|
||||
|
||||
public void setEncording(String encording){this.encording = encording;}
|
||||
|
||||
private boolean needPassword = false;
|
||||
private boolean useZip4jDecryption = false;
|
||||
|
||||
@@ -58,8 +55,6 @@ public class Unzip {
|
||||
|
||||
private String sourceZip;
|
||||
private String targetDir;
|
||||
private String encording;
|
||||
|
||||
|
||||
public Unzip(String sourceZip, String targetDir) {
|
||||
this.sourceZip = sourceZip;
|
||||
@@ -97,9 +92,6 @@ public class Unzip {
|
||||
}
|
||||
|
||||
ZipFile zipFile = new ZipFile(sourceZip);
|
||||
if(encording != null){
|
||||
zipFile.setFileNameCharset(encording);
|
||||
}
|
||||
|
||||
if (checkArchive) {
|
||||
if (!zipFile.isValidZipFile()) {
|
||||
@@ -160,8 +152,9 @@ public class Unzip {
|
||||
is = new javax.crypto.CipherInputStream(is,
|
||||
org.talend.archive.IntegrityUtil.createCipher(
|
||||
javax.crypto.Cipher.DECRYPT_MODE, password));
|
||||
|
||||
org.apache.commons.compress.archivers.zip.ZipArchiveInputStream input = new org.apache.commons.compress.archivers.zip.ZipArchiveInputStream(
|
||||
new java.io.BufferedInputStream(is),Optional.ofNullable(encording).orElse("UTF8"));
|
||||
new java.io.BufferedInputStream(is));
|
||||
org.apache.commons.compress.archivers.zip.ZipArchiveEntry entry;
|
||||
|
||||
while ((entry = input.getNextZipEntry()) != null) {
|
||||
@@ -197,7 +190,7 @@ public class Unzip {
|
||||
org.apache.commons.compress.archivers.zip.ZipFile zip = null;
|
||||
try {
|
||||
zip = new org.apache.commons.compress.archivers.zip.ZipFile(
|
||||
sourceZip,Optional.ofNullable(encording).orElse("UTF8"));
|
||||
sourceZip);
|
||||
java.util.Enumeration enuFiles = zip.getEntries();
|
||||
java.io.InputStream is = null;
|
||||
|
||||
|
||||
@@ -132,7 +132,7 @@ int tmp_batchUpdateCount_<%=cid%> = 0;
|
||||
int deletedCount_<%=cid%>=0;
|
||||
int updatedCount_<%=cid%>=0;
|
||||
int insertedCount_<%=cid%>=0;
|
||||
|
||||
int rowsToCommitCount_<%=cid%>=0;
|
||||
String tableName_<%=cid%> = <%=table%>;
|
||||
boolean whetherReject_<%=cid%> = false;
|
||||
|
||||
|
||||
@@ -102,10 +102,12 @@ imports="
|
||||
if(("true").equals(useCommitControl)) {
|
||||
if(!("").equals(commitEvery) && !("0").equals(commitEvery)) {
|
||||
%>
|
||||
if(commitCounter_<%=cid%> > 0) {
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
if(commitCounter_<%=cid%> > 0 && rowsToCommitCount_<%=cid%> != 0) {
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
commitCounter_<%=cid%> = 0;
|
||||
}
|
||||
<%
|
||||
}
|
||||
|
||||
@@ -310,7 +310,9 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.sql.SQLException e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
@@ -442,7 +444,9 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
try {
|
||||
nb_line_<%=cid%>++;
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.sql.SQLException e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
@@ -589,7 +593,9 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
int processedCount_<%=cid%> = pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.sql.SQLException e) {
|
||||
@@ -665,7 +671,9 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
int processedCount_<%=cid%> = pstmtInsert_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.sql.SQLException e) {
|
||||
@@ -795,6 +803,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
updateFlag_<%=cid%>=pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%>+updateFlag_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += updateFlag_<%=cid%>;
|
||||
|
||||
if(updateFlag_<%=cid%> == 0) {
|
||||
|
||||
@@ -840,7 +849,9 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
%>
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
int processedCount_<%=cid%> = pstmtInsert_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
}else{
|
||||
@@ -930,7 +941,9 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
deletedCount_<%=cid%> = deletedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
deletedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().deleting(dbLog.var("nb_line"));%>
|
||||
} catch(java.sql.SQLException e) {
|
||||
@@ -1014,10 +1027,13 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
}
|
||||
<%
|
||||
}%>
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
commitCounter_<%=cid%>=0;
|
||||
if(rowsToCommitCount_<%=cid%> != 0){
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
}
|
||||
commitCounter_<%=cid%>=0;
|
||||
}
|
||||
<%
|
||||
}
|
||||
|
||||
@@ -118,7 +118,7 @@
|
||||
<IMPORT NAME="Driver-JACKCESS" MODULE="jackcess-2.1.12.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess/2.1.12" REQUIRED="true"/>
|
||||
|
||||
<IMPORT NAME="jackcess-encrypt-2.1.4" MODULE="jackcess-encrypt-2.1.4.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess-encrypt/2.1.4" REQUIRED="true"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.68" MODULE="bcprov-jdk15on-1.68.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.68" REQUIRED="true"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.60" MODULE="bcprov-jdk15on-1.60.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.60" REQUIRED="true"/>
|
||||
<IMPORT NAME="talend-ucanaccess-utils-1.0.0" MODULE="talend-ucanaccess-utils-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-ucanaccess-utils-1.0.0/6.4.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.access/lib/talend-ucanaccess-utils-1.0.0.jar" REQUIRED="true"/>
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED="true"/>
|
||||
|
||||
@@ -121,7 +121,7 @@
|
||||
<IMPORT NAME="Driver-JACKCESS" MODULE="jackcess-2.1.12.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess/2.1.12" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="jackcess-encrypt-2.1.4" MODULE="jackcess-encrypt-2.1.4.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess-encrypt/2.1.4" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.68" MODULE="bcprov-jdk15on-1.68.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.68" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.60" MODULE="bcprov-jdk15on-1.60.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.60" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="talend-ucanaccess-utils-1.0.0" MODULE="talend-ucanaccess-utils-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-ucanaccess-utils-1.0.0/6.4.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.access/lib/talend-ucanaccess-utils-1.0.0.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
@@ -100,7 +100,7 @@ int nb_line_rejected_<%=cid%> = 0;
|
||||
int deletedCount_<%=cid%>=0;
|
||||
int updatedCount_<%=cid%>=0;
|
||||
int insertedCount_<%=cid%>=0;
|
||||
|
||||
int rowsToCommitCount_<%=cid%>=0;
|
||||
String tableName_<%=cid%> = <%=tableName%>;
|
||||
boolean whetherReject_<%=cid%> = false;
|
||||
java.sql.Connection conn_<%=cid%> = null;
|
||||
|
||||
@@ -61,9 +61,13 @@ imports="
|
||||
{
|
||||
if(!("").equals(commitEvery)&&!("0").equals(commitEvery)){
|
||||
%>
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
if(rowsToCommitCount_<%=cid%> != 0){
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
}
|
||||
commitCounter_<%=cid%> = 0;
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -207,7 +207,7 @@
|
||||
<IMPORT NAME="Driver-JACKCESS" MODULE="jackcess-2.1.12.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess/2.1.12" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="jackcess-encrypt-2.1.4" MODULE="jackcess-encrypt-2.1.4.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess-encrypt/2.1.4" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.68" MODULE="bcprov-jdk15on-1.68.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.68" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.60" MODULE="bcprov-jdk15on-1.60.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.60" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="talend-ucanaccess-utils-1.0.0" MODULE="talend-ucanaccess-utils-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-ucanaccess-utils-1.0.0/6.4.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.access/lib/talend-ucanaccess-utils-1.0.0.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
@@ -140,7 +140,9 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -230,7 +232,9 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -348,7 +352,9 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%> + pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
int processedCount_<%=cid%> = pstmtUpdate_<%=cid %>.executeUpdate();
|
||||
updatedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -410,7 +416,9 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try{
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
int processedCount_<%=cid%> = pstmtInsert_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> = processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -511,7 +519,8 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().updating(dbLog.var("nb_line"));%>
|
||||
}
|
||||
updatedCount_<%=cid%> = updatedCount_<%=cid%>+updateFlag_<%=cid%>;
|
||||
updatedCount_<%=cid%> += updateFlag_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += updateFlag_<%=cid%>;
|
||||
} catch(java.lang.Exception e) {
|
||||
whetherReject_<%=cid%> = true;
|
||||
<%
|
||||
@@ -573,7 +582,9 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
insertedCount_<%=cid%> = insertedCount_<%=cid%> + pstmtInsert_<%=cid %>.executeUpdate();
|
||||
int processedCount_<%=cid%> = pstmtInsert_<%=cid %>.executeUpdate();
|
||||
insertedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
rowsToCommitCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().inserting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -585,7 +596,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
nb_line_<%=cid%>++;
|
||||
nb_line_<%=cid%>++;
|
||||
<%
|
||||
if(rejectConnName != null && rejectColumnList != null && rejectColumnList.size() > 0) {
|
||||
%>
|
||||
@@ -652,7 +663,8 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
try {
|
||||
<%if(isEnableDebug){dbLog.data().sqlDebugPrint("globalMap.get(\""+cid+"_QUERY\")");}%>
|
||||
deletedCount_<%=cid%> = deletedCount_<%=cid%> + pstmt_<%=cid %>.executeUpdate();
|
||||
int processedCount_<%=cid%> = pstmt_<%=cid %>.executeUpdate();
|
||||
deletedCount_<%=cid%> += processedCount_<%=cid%>;
|
||||
nb_line_<%=cid%>++;
|
||||
<%dbLog.data().deleting(dbLog.var("nb_line"));%>
|
||||
} catch(java.lang.Exception e) {
|
||||
@@ -697,7 +709,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
<%
|
||||
for(IConnection outgoingConn : outgoingConns) {
|
||||
if(rejectConnName == null || (rejectConnName != null && !outgoingConn.getName().equals(rejectConnName))) {
|
||||
if(outgoingConn.getLineStyle().hasConnectionCategory(IConnectionCategory.DATA)) {
|
||||
if(outgoingConn.getLineStyle().hasConnectionCategory(IConnectionCategory.DATA)){
|
||||
%>
|
||||
<%=outgoingConn.getName()%> = new <%=outgoingConn.getName()%>Struct();
|
||||
<%
|
||||
@@ -720,12 +732,15 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
%>
|
||||
commitCounter_<%=cid%>++;
|
||||
|
||||
if(commitEvery_<%=cid%><=commitCounter_<%=cid%>) {
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
commitCounter_<%=cid%>=0;
|
||||
}
|
||||
if(commitEvery_<%=cid%> <= commitCounter_<%=cid%>){
|
||||
if(rowsToCommitCount_<%=cid%> != 0){
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
}
|
||||
commitCounter_<%=cid%>=0;
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,7 +181,7 @@
|
||||
<IMPORT NAME="Driver-JACKCESS" MODULE="jackcess-2.1.12.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess/2.1.12" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="jackcess-encrypt-2.1.4" MODULE="jackcess-encrypt-2.1.4.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess-encrypt/2.1.4" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.68" MODULE="bcprov-jdk15on-1.68.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.68" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.60" MODULE="bcprov-jdk15on-1.60.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.60" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="talend-ucanaccess-utils-1.0.0" MODULE="talend-ucanaccess-utils-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-ucanaccess-utils-1.0.0/6.4.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.access/lib/talend-ucanaccess-utils-1.0.0.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
@@ -211,7 +211,7 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Java_DOM4J" MODULE="dom4j-2.1.3.jar" MVN="mvn:org.dom4j/dom4j/2.1.3" REQUIRED="true" BundleID="" />
|
||||
<IMPORT NAME="Java_DOM4J_2.1.1" MODULE="dom4j-2.1.1.jar" MVN="mvn:org.dom4j/dom4j/2.1.1" REQUIRED="true" BundleID="" />
|
||||
<IMPORT NAME="Java_JAXEN_1.1.6" MODULE="jaxen-1.1.6.jar" MVN="mvn:jaxen/jaxen/1.1.6" REQUIRED="true" BundleID="" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -228,23 +228,23 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="aws-java-sdk-1.11.848.jar" MODULE="aws-java-sdk-1.11.848.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.848"
|
||||
<IMPORT NAME="aws-java-sdk-1.11.729.jar" MODULE="aws-java-sdk-1.11.729.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.729"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.11.4.jar" MODULE="jackson-core-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind-2.11.4.jar" MODULE="jackson-databind-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
<IMPORT NAME="jackson-databind-2.10.1.jar" MODULE="jackson-databind-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations-2.11.4.jar" MODULE="jackson-annotations-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
<IMPORT NAME="jackson-annotations-2.10.1.jar" MODULE="jackson-annotations-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
<IMPORT NAME="httpcore-4.4.11.jar" MODULE="httpcore-4.4.11.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.11"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpclient-4.5.12.jar" MODULE="httpclient-4.5.12.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpclient/4.5.12"
|
||||
<IMPORT NAME="httpclient-4.5.9.jar" MODULE="httpclient-4.5.9.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpclient/4.5.9"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="joda-time-2.8.1.jar" MODULE="joda-time-2.8.1.jar"
|
||||
MVN="mvn:joda-time/joda-time/2.8.1"
|
||||
@@ -252,8 +252,8 @@
|
||||
<IMPORT NAME="commons-logging-1.2.jar" MODULE="commons-logging-1.2.jar"
|
||||
MVN="mvn:commons-logging/commons-logging/1.2"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="commons-codec-1.14.jar" MODULE="commons-codec-1.14.jar"
|
||||
MVN="mvn:commons-codec/commons-codec/1.14"
|
||||
<IMPORT NAME="commons-codec-1.11.jar" MODULE="commons-codec-1.11.jar"
|
||||
MVN="mvn:commons-codec/commons-codec/1.11"
|
||||
REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -110,7 +110,6 @@
|
||||
String key_pair = ElementParameterParser.getValue(node, "__KEY_PAIR__");
|
||||
|
||||
String instance_count = ElementParameterParser.getValue(node, "__INSTANCE_COUNT__");
|
||||
String slave_instance_count = ElementParameterParser.getValue(node, "__SLAVE_INSTANCE_COUNT__");
|
||||
|
||||
String master_instance_type = ElementParameterParser.getValue(node, "__MASTER_INSTANCE_TYPE__");
|
||||
String slave_instance_type = ElementParameterParser.getValue(node, "__SLAVE_INSTANCE_TYPE__");
|
||||
@@ -140,8 +139,6 @@
|
||||
boolean use_customize_version = "true".equals(ElementParameterParser.getValue(node, "__USE_CUSTOMIZE_VERSION__"));
|
||||
String cluster_version = use_customize_version?ElementParameterParser.getValue(node, "__CUSTOMIZE_VERSION__").trim().replace(' ', '-'):ElementParameterParser.getValue(node, "__CLUSTER_VERSION__");
|
||||
cluster_version = cluster_version.toLowerCase();
|
||||
|
||||
boolean use_multiple_master = "true".equals(ElementParameterParser.getValue(node, "__USE_MULTIPLE_MASTER_NODES__"));
|
||||
%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/Amazon/EMRClient.javajet"%>
|
||||
@@ -249,81 +246,59 @@
|
||||
if(setApps) {
|
||||
%>
|
||||
<%@ include file="./app_mapping.javajet"%>
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
|
||||
java.util.List<com.amazonaws.services.elasticmapreduce.model.Configuration> configs_<%=cid%> = new java.util.ArrayList<com.amazonaws.services.elasticmapreduce.model.Configuration>();
|
||||
<%
|
||||
Map<String, List<Map<String,String>>> classification2Properties = new HashMap<String, List<Map<String,String>>>();
|
||||
String property_json = ElementParameterParser.getValue(node, "__PROPERTY_JSON__");
|
||||
boolean property_json_validated = property_json != null && !property_json.equals("\"\"") && property_json.length() > 0;
|
||||
%>
|
||||
boolean property_json_validated_<%=cid%> = <%=property_json_validated %>;
|
||||
<%
|
||||
if(!use_multiple_master) {
|
||||
if(newApi) {
|
||||
List<Map<String, String>> property_table = (List<Map<String, String>>)ElementParameterParser.getObjectValue(node, "__PROPERTY_TABLE__");
|
||||
if(property_table!=null) {
|
||||
for (int i=0; i<property_table.size(); i++) {
|
||||
Map<String, String> property = property_table.get(i);
|
||||
String classification = property.get("CLASSIFICATION");
|
||||
|
||||
List<Map<String,String>> properties = classification2Properties.get(classification);
|
||||
if(properties == null) {
|
||||
properties = new ArrayList<Map<String,String>>();
|
||||
classification2Properties.put(classification, properties);
|
||||
}
|
||||
|
||||
properties.add(property);
|
||||
if(newApi) {
|
||||
List<Map<String, String>> property_table = (List<Map<String, String>>)ElementParameterParser.getObjectValue(node, "__PROPERTY_TABLE__");
|
||||
if(property_table!=null) {
|
||||
for (int i=0; i<property_table.size(); i++) {
|
||||
Map<String, String> property = property_table.get(i);
|
||||
String classification = property.get("CLASSIFICATION");
|
||||
|
||||
List<Map<String,String>> properties = classification2Properties.get(classification);
|
||||
if(properties == null) {
|
||||
properties = new ArrayList<Map<String,String>>();
|
||||
classification2Properties.put(classification, properties);
|
||||
}
|
||||
|
||||
if(!classification2Properties.isEmpty()) {
|
||||
properties.add(property);
|
||||
}
|
||||
|
||||
if(!classification2Properties.isEmpty()) {
|
||||
%>
|
||||
java.util.List<com.amazonaws.services.elasticmapreduce.model.Configuration> configs_<%=cid%> = new java.util.ArrayList<com.amazonaws.services.elasticmapreduce.model.Configuration>();
|
||||
java.util.Map<String,String> properties_<%=cid%> = null;
|
||||
com.amazonaws.services.elasticmapreduce.model.Configuration config_<%=cid%> = null;
|
||||
<%
|
||||
}
|
||||
|
||||
for(java.util.Map.Entry<String, List<Map<String,String>>> entry : classification2Properties.entrySet()) {
|
||||
String classification = entry.getKey();
|
||||
List<Map<String,String>> properties = entry.getValue();
|
||||
%>
|
||||
java.util.Map<String,String> properties_<%=cid%> = null;
|
||||
com.amazonaws.services.elasticmapreduce.model.Configuration config_<%=cid%> = null;
|
||||
<%
|
||||
}
|
||||
|
||||
for(java.util.Map.Entry<String, List<Map<String,String>>> entry : classification2Properties.entrySet()) {
|
||||
String classification = entry.getKey();
|
||||
List<Map<String,String>> properties = entry.getValue();
|
||||
%>
|
||||
properties_<%=cid%> = new java.util.HashMap<String,String>();
|
||||
|
||||
<%
|
||||
for(Map<String,String> property : properties) {
|
||||
%>
|
||||
properties_<%=cid%>.put(<%=property.get("KEY")%>, <%=property.get("VALUE")%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
config_<%=cid%> = new com.amazonaws.services.elasticmapreduce.model.Configuration()
|
||||
.withClassification(<%=classification%>)
|
||||
.withProperties(properties_<%=cid%>);
|
||||
|
||||
configs_<%=cid%>.add(config_<%=cid%>);
|
||||
<%
|
||||
}
|
||||
properties_<%=cid%> = new java.util.HashMap<String,String>();
|
||||
|
||||
<%
|
||||
for(Map<String,String> property : properties) {
|
||||
%>
|
||||
properties_<%=cid%>.put(<%=property.get("KEY")%>, <%=property.get("VALUE")%>);
|
||||
<%
|
||||
}
|
||||
%>
|
||||
|
||||
config_<%=cid%> = new com.amazonaws.services.elasticmapreduce.model.Configuration()
|
||||
.withClassification(<%=classification%>)
|
||||
.withProperties(properties_<%=cid%>);
|
||||
|
||||
configs_<%=cid%>.add(config_<%=cid%>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
}else{
|
||||
boolean isHasSubnetId = subnet_id != null && !subnet_id.equals("\"\"") && subnet_id.length() > 0;
|
||||
}
|
||||
%>
|
||||
boolean isHasSubnetId_<%=cid%> = <%=isHasSubnetId %>;
|
||||
if(!isHasSubnetId_<%=cid%>) {
|
||||
throw new RuntimeException("The subnet ID must be specified while using multiple master nodes.");
|
||||
}
|
||||
|
||||
if(property_json_validated_<%=cid%>) {
|
||||
com.fasterxml.jackson.databind.ObjectMapper mapper_<%=cid%> = new com.fasterxml.jackson.databind.ObjectMapper();
|
||||
mapper_<%=cid%>.configure(com.fasterxml.jackson.databind.MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true);
|
||||
try {
|
||||
configs_<%=cid%> = java.util.Arrays.asList(mapper_<%=cid%>.readValue(<%=property_json%>, com.amazonaws.services.elasticmapreduce.model.Configuration[].class));
|
||||
} catch (com.fasterxml.jackson.core.JsonProcessingException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
<%}%>
|
||||
|
||||
com.amazonaws.services.elasticmapreduce.model.RunJobFlowRequest request_<%=cid%> = new com.amazonaws.services.elasticmapreduce.model.RunJobFlowRequest()
|
||||
.withName(<%=cluster_name%>)
|
||||
<%if(enable_log){%>
|
||||
@@ -367,32 +342,18 @@
|
||||
.withBootstrapActions(actions_<%=cid %>)
|
||||
<%}%>
|
||||
|
||||
<%if(!classification2Properties.isEmpty() || (use_multiple_master && property_json_validated)){%>
|
||||
.withConfigurations(configs_<%=cid%>)
|
||||
<%}%>
|
||||
<%if(!classification2Properties.isEmpty()){%>
|
||||
.withConfigurations(configs_<%=cid%>)
|
||||
<%}%>
|
||||
|
||||
.withInstances(
|
||||
new com.amazonaws.services.elasticmapreduce.model.JobFlowInstancesConfig()
|
||||
.withInstanceCount(<%=instance_count%>)
|
||||
<%if(enable_key_pair){%>
|
||||
.withEc2KeyName(<%=key_pair%>)
|
||||
<%}%>
|
||||
|
||||
<%if(use_multiple_master){%>
|
||||
.withInstanceGroups(
|
||||
new com.amazonaws.services.elasticmapreduce.model.InstanceGroupConfig()
|
||||
.withInstanceRole(com.amazonaws.services.elasticmapreduce.model.InstanceRoleType.MASTER)
|
||||
.withInstanceCount(3)
|
||||
.withInstanceType(<%=master_instance_type%>),
|
||||
new com.amazonaws.services.elasticmapreduce.model.InstanceGroupConfig()
|
||||
.withInstanceRole(com.amazonaws.services.elasticmapreduce.model.InstanceRoleType.CORE)
|
||||
.withInstanceCount(<%=slave_instance_count%>)
|
||||
.withInstanceType(<%=slave_instance_type%>)
|
||||
)
|
||||
<%}else{%>
|
||||
.withInstanceCount(<%=instance_count%>)
|
||||
.withMasterInstanceType(<%=master_instance_type%>)
|
||||
.withSlaveInstanceType(<%=slave_instance_type%>)
|
||||
<%}%>
|
||||
|
||||
<%if(parameterUtil.isValid(subnet_id)){%>
|
||||
.withEc2SubnetId(<%=subnet_id%>)
|
||||
|
||||
@@ -157,14 +157,9 @@
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="INSTANCE_COUNT" FIELD="TEXT" NUM_ROW="52"
|
||||
SHOW_IF="ACTION=='START' AND (USE_MULTIPLE_MASTER_NODES=='false')" GROUP="INSTANCE_CONFIGURATION">
|
||||
SHOW_IF="ACTION=='START'" GROUP="INSTANCE_CONFIGURATION">
|
||||
<DEFAULT>3</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="SLAVE_INSTANCE_COUNT" FIELD="TEXT" NUM_ROW="52"
|
||||
SHOW_IF="ACTION=='START' AND (USE_MULTIPLE_MASTER_NODES=='true')" GROUP="INSTANCE_CONFIGURATION">
|
||||
<DEFAULT>2</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="MASTER_INSTANCE_TYPE" FIELD="OPENED_LIST"
|
||||
NUM_ROW="55" SHOW_IF="ACTION=='START'" GROUP="INSTANCE_CONFIGURATION">
|
||||
@@ -410,16 +405,6 @@
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="USE_MULTIPLE_MASTER_NODES" FIELD="CHECK"
|
||||
NUM_ROW="31" SHOW_IF="ACTION=='START'" GROUP="ADVANCED_CONFIGURATION">
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="NOTE" FIELD="LABEL"
|
||||
NUM_ROW="31" SHOW_IF="isShow[USE_MULTIPLE_MASTER_NODES]" GROUP="ADVANCED_CONFIGURATION" >
|
||||
<DEFAULT>(supported by version 5.23.0 and later)</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="CUSTOMIZE_VERSION" FIELD="TEXT"
|
||||
NUM_ROW="35" SHOW_IF="(ACTION=='START') AND (USE_CUSTOMIZE_VERSION =='true')" GROUP="CUSTOMIZE_APPLICATION_GROUP">
|
||||
<DEFAULT>"emr-5.29.0"</DEFAULT>
|
||||
@@ -537,41 +522,34 @@
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="PROPERTY_TABLE" FIELD="TABLE" NUM_ROW="95" NB_LINES="5" GROUP="PROPERTY_GROUP"
|
||||
SHOW_IF="(ACTION=='START') AND (CLUSTER_VERSION!='AMI3_10_0') AND (CLUSTER_VERSION!='AMI3_9_0') AND (CLUSTER_VERSION!='AMI3_8_0') AND (CLUSTER_VERSION!='AMI2_4_11') AND (CLUSTER_VERSION!='AMI2_4_10')
|
||||
AND (USE_MULTIPLE_MASTER_NODES=='false')">
|
||||
SHOW_IF="(ACTION=='START') AND (CLUSTER_VERSION!='AMI3_10_0') AND (CLUSTER_VERSION!='AMI3_9_0') AND (CLUSTER_VERSION!='AMI3_8_0') AND (CLUSTER_VERSION!='AMI2_4_11') AND (CLUSTER_VERSION!='AMI2_4_10')">
|
||||
<ITEMS>
|
||||
<ITEM NAME="CLASSIFICATION" />
|
||||
<ITEM NAME="KEY" />
|
||||
<ITEM NAME="VALUE" />
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER NAME="PROPERTY_JSON" FIELD="MEMO" NUM_ROW="95" NB_LINES="15" GROUP="PROPERTY_GROUP"
|
||||
SHOW_IF="(ACTION=='START') AND (CLUSTER_VERSION!='AMI3_10_0') AND (CLUSTER_VERSION!='AMI3_9_0') AND (CLUSTER_VERSION!='AMI3_8_0') AND (CLUSTER_VERSION!='AMI2_4_11') AND (CLUSTER_VERSION!='AMI2_4_10')
|
||||
AND (USE_MULTIPLE_MASTER_NODES=='true')">
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
</ADVANCED_PARAMETERS>
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="aws-java-sdk-1.11.848.jar" MODULE="aws-java-sdk-1.11.848.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.848"
|
||||
<IMPORT NAME="aws-java-sdk-1.11.729.jar" MODULE="aws-java-sdk-1.11.729.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.729"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.11.4.jar" MODULE="jackson-core-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind-2.11.4.jar" MODULE="jackson-databind-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
<IMPORT NAME="jackson-databind-2.10.1.jar" MODULE="jackson-databind-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations-2.11.4.jar" MODULE="jackson-annotations-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
<IMPORT NAME="jackson-annotations-2.10.1.jar" MODULE="jackson-annotations-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
<IMPORT NAME="httpcore-4.4.11.jar" MODULE="httpcore-4.4.11.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.11"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpclient-4.5.12.jar" MODULE="httpclient-4.5.12.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpclient/4.5.12"
|
||||
<IMPORT NAME="httpclient-4.5.9.jar" MODULE="httpclient-4.5.9.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpclient/4.5.9"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="joda-time-2.8.1.jar" MODULE="joda-time-2.8.1.jar"
|
||||
MVN="mvn:joda-time/joda-time/2.8.1"
|
||||
@@ -579,8 +557,8 @@
|
||||
<IMPORT NAME="commons-logging-1.2.jar" MODULE="commons-logging-1.2.jar"
|
||||
MVN="mvn:commons-logging/commons-logging/1.2"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="commons-codec-1.14.jar" MODULE="commons-codec-1.14.jar"
|
||||
MVN="mvn:commons-codec/commons-codec/1.14"
|
||||
<IMPORT NAME="commons-codec-1.11.jar" MODULE="commons-codec-1.11.jar"
|
||||
MVN="mvn:commons-codec/commons-codec/1.11"
|
||||
REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -62,8 +62,6 @@ INSTANCE_CONFIGURATION.NAME=Instance Configuration
|
||||
|
||||
INSTANCE_COUNT.NAME=Instance count
|
||||
|
||||
SLAVE_INSTANCE_COUNT.NAME=Slave instance count
|
||||
|
||||
MASTER_INSTANCE_TYPE.NAME=Master instance type
|
||||
MASTER_INSTANCE_TYPE.ITEM.m3.xlarge=m3.xlarge
|
||||
MASTER_INSTANCE_TYPE.ITEM.m3.2xlarge=m3.2xlarge
|
||||
@@ -193,12 +191,8 @@ PROPERTY_TABLE.ITEM.CLASSIFICATION=Classification
|
||||
PROPERTY_TABLE.ITEM.KEY=Key
|
||||
PROPERTY_TABLE.ITEM.VALUE=Value
|
||||
|
||||
PROPERTY_JSON.NAME=Properties in JSON
|
||||
|
||||
ENABLE_DEBUG.NAME=Enable debug
|
||||
|
||||
USE_MULTIPLE_MASTER_NODES.NAME=Use multiple master nodes
|
||||
|
||||
INHERIT_CREDENTIALS.NAME=Inherit credentials from AWS role
|
||||
ASSUME_ROLE.NAME=Assume role
|
||||
ARN.NAME=Role ARN
|
||||
|
||||
@@ -285,23 +285,23 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="aws-java-sdk-1.11.848.jar" MODULE="aws-java-sdk-1.11.848.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.848"
|
||||
<IMPORT NAME="aws-java-sdk-1.11.729.jar" MODULE="aws-java-sdk-1.11.729.jar"
|
||||
MVN="mvn:com.amazonaws/aws-java-sdk/1.11.729"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-core-2.11.4.jar" MODULE="jackson-core-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.11.4"
|
||||
<IMPORT NAME="jackson-core-2.10.1.jar" MODULE="jackson-core-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-core/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-databind-2.11.4.jar" MODULE="jackson-databind-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.11.4"
|
||||
<IMPORT NAME="jackson-databind-2.10.1.jar" MODULE="jackson-databind-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-databind/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="jackson-annotations-2.11.4.jar" MODULE="jackson-annotations-2.11.4.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.11.4"
|
||||
<IMPORT NAME="jackson-annotations-2.10.1.jar" MODULE="jackson-annotations-2.10.1.jar"
|
||||
MVN="mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.1"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
<IMPORT NAME="httpcore-4.4.11.jar" MODULE="httpcore-4.4.11.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpcore/4.4.11"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="httpclient-4.5.12.jar" MODULE="httpclient-4.5.12.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpclient/4.5.12"
|
||||
<IMPORT NAME="httpclient-4.5.9.jar" MODULE="httpclient-4.5.9.jar"
|
||||
MVN="mvn:org.apache.httpcomponents/httpclient/4.5.9"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="joda-time-2.8.1.jar" MODULE="joda-time-2.8.1.jar"
|
||||
MVN="mvn:joda-time/joda-time/2.8.1"
|
||||
@@ -309,8 +309,8 @@
|
||||
<IMPORT NAME="commons-logging-1.2.jar" MODULE="commons-logging-1.2.jar"
|
||||
MVN="mvn:commons-logging/commons-logging/1.2"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="commons-codec-1.14.jar" MODULE="commons-codec-1.14.jar"
|
||||
MVN="mvn:commons-codec/commons-codec/1.14"
|
||||
<IMPORT NAME="commons-codec-1.11.jar" MODULE="commons-codec-1.11.jar"
|
||||
MVN="mvn:commons-codec/commons-codec/1.11"
|
||||
REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -163,7 +163,7 @@ int nb_line_rejected_<%=cid%> = 0;
|
||||
int deletedCount_<%=cid%>=0;
|
||||
int updatedCount_<%=cid%>=0;
|
||||
int insertedCount_<%=cid%>=0;
|
||||
|
||||
int rowsToCommitCount_<%=cid%>=0;
|
||||
int rejectedCount_<%=cid%>=0;
|
||||
|
||||
String tableName_<%=cid%> = <%=table%>;
|
||||
|
||||
@@ -37,7 +37,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
String dataAction = ElementParameterParser.getValue(node,"__DATA_ACTION__");
|
||||
|
||||
String commitEvery = ElementParameterParser.getValue(node, "__COMMIT_EVERY__");
|
||||
|
||||
|
||||
boolean setAutoCommit = false;
|
||||
|
||||
boolean useExistingConnection = "true".equals(ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__"));
|
||||
@@ -121,6 +121,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
for(int countEach_<%=cid%>: pstmt_<%=cid %>.executeBatch()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : 1);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%dbLog.batch().executeDone(dbLog.str(dataAction));%>
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
@@ -141,6 +142,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
for(int countEach_<%=cid%>: e.getUpdateCounts()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : countEach_<%=cid%>);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
insertedCount_<%=cid%> += countSum_<%=cid%>;
|
||||
@@ -196,6 +198,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
for(int countEach_<%=cid%>: pstmt_<%=cid %>.executeBatch()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : <%=("INSERT").equals(dataAction)? "1" : "countEach_" +cid%>);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%dbLog.batch().executeDone(dbLog.str(dataAction));%>
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
@@ -221,6 +224,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
for(int countEach_<%=cid%>: e.getUpdateCounts()) {
|
||||
countSum_<%=cid%> += (countEach_<%=cid%> < 0 ? 0 : countEach_<%=cid%>);
|
||||
}
|
||||
rowsToCommitCount_<%=cid%> += countSum_<%=cid%>;
|
||||
<%if (("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
insertedCount_<%=cid%> += countSum_<%=cid%>;
|
||||
@@ -254,10 +258,12 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
if(!("").equals(commitEvery) && !("0").equals(commitEvery))
|
||||
{
|
||||
%>
|
||||
if (commitCounter_<%=cid%> > 0) {
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("commitCounter"));%>
|
||||
if (commitCounter_<%=cid%> > 0 && rowsToCommitCount_<%=cid%> != 0) {
|
||||
<%dbLog.commit().commitTry(null, dbLog.var("rowsToCommitCount"));%>
|
||||
conn_<%=cid%>.commit();
|
||||
<%dbLog.commit().commitDone(null);%>
|
||||
rowsToCommitCount_<%=cid%> = 0;
|
||||
commitCounter_<%=cid%> = 0;
|
||||
}
|
||||
<%
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user