Compare commits
145 Commits
patch/7.3.
...
patch/8.0.
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
893b214fcf | ||
|
|
c0d3e099dd | ||
|
|
3970eb25af | ||
|
|
79361417eb | ||
|
|
1240c228c1 | ||
|
|
47953ea772 | ||
|
|
8bf40999cb | ||
|
|
d8824c7d80 | ||
|
|
c89e5a35db | ||
|
|
22b2899392 | ||
|
|
5c5e4a561f | ||
|
|
a972cbe1ec | ||
|
|
79599d9db9 | ||
|
|
e0c0d31522 | ||
|
|
bc6944ddf6 | ||
|
|
93afe3609d | ||
|
|
6aca900a7e | ||
|
|
bfee355f0a | ||
|
|
a5e1cc47e7 | ||
|
|
1d1adfc74c | ||
|
|
86556e52fe | ||
|
|
2e3af5d1d1 | ||
|
|
eacf6b2edc | ||
|
|
f8db539d1d | ||
|
|
9041181162 | ||
|
|
3379f247d4 | ||
|
|
0f587babf5 | ||
|
|
21135a3514 | ||
|
|
483a0f4308 | ||
|
|
9580f89ac6 | ||
|
|
816212547d | ||
|
|
b67a5e2bce | ||
|
|
b2f1487ae9 | ||
|
|
8c01f53dd0 | ||
|
|
f057c13965 | ||
|
|
d0bd3a8de2 | ||
|
|
529bb5a138 | ||
|
|
bc920a8e4a | ||
|
|
a4f1afa7d9 | ||
|
|
240510aa13 | ||
|
|
4ea4723304 | ||
|
|
05e1edbbc1 | ||
|
|
1eebeae0c2 | ||
|
|
122af47d85 | ||
|
|
6f373d1522 | ||
|
|
b6161bdc13 | ||
|
|
df7fd9386e | ||
|
|
bf6cbaf984 | ||
|
|
c8b5f40e43 | ||
|
|
4e39ebf917 | ||
|
|
106244a15f | ||
|
|
df42fc827b | ||
|
|
9d12d4b08a | ||
|
|
a6c5e1f537 | ||
|
|
06462122bb | ||
|
|
2c3eafb7c2 | ||
|
|
eef575ea44 | ||
|
|
7be4c6a799 | ||
|
|
f852743538 | ||
|
|
f590b9214d | ||
|
|
3544beccec | ||
|
|
195c2997d2 | ||
|
|
74f1c89f0b | ||
|
|
9134549c94 | ||
|
|
9e472b477a | ||
|
|
41516246d1 | ||
|
|
f9fc607e59 | ||
|
|
59ace742af | ||
|
|
1e35baefb1 | ||
|
|
bbe4460cd4 | ||
|
|
35e6bf01f9 | ||
|
|
4191ba8730 | ||
|
|
22cad3b97a | ||
|
|
87fbabdccd | ||
|
|
b62c16ff6a | ||
|
|
32408cd9a7 | ||
|
|
94a80b55d4 | ||
|
|
c7a9cc1145 | ||
|
|
85ed098bcb | ||
|
|
25637d3857 | ||
|
|
b3774b643b | ||
|
|
51d6fb4cac | ||
|
|
37ae765116 | ||
|
|
1b891d23f8 | ||
|
|
26c3b77921 | ||
|
|
6af4903291 | ||
|
|
f3a1279436 | ||
|
|
667e43c56e | ||
|
|
6120adbd1e | ||
|
|
63dae01a82 | ||
|
|
52b46db595 | ||
|
|
5cac10311a | ||
|
|
fc995fd934 | ||
|
|
0c1aa7d269 | ||
|
|
58cb31cd0d | ||
|
|
2de786b6db | ||
|
|
4aa47a90a9 | ||
|
|
018e3e3e06 | ||
|
|
69168c56b8 | ||
|
|
cfb02c57c3 | ||
|
|
1b20a2d08c | ||
|
|
d8af56e14f | ||
|
|
23070c60a1 | ||
|
|
7278437430 | ||
|
|
0d1d63b882 | ||
|
|
732e383f8e | ||
|
|
c7e01ebe67 | ||
|
|
4b365b194b | ||
|
|
7f7c963cdc | ||
|
|
586bcb9d23 | ||
|
|
bac3605a26 | ||
|
|
c37faee0d1 | ||
|
|
bca1ab75b5 | ||
|
|
e03c026b74 | ||
|
|
342a7350be | ||
|
|
4cc1dd3de9 | ||
|
|
ffb98f3f6f | ||
|
|
62c2e341c5 | ||
|
|
d22a213e38 | ||
|
|
5a3fd2ef23 | ||
|
|
f6448c1316 | ||
|
|
b863480a14 | ||
|
|
19a9126382 | ||
|
|
8e9540b70b | ||
|
|
8c0a3390a6 | ||
|
|
9eac6bc883 | ||
|
|
37a0af7c4e | ||
|
|
167f9aa41e | ||
|
|
250580cddd | ||
|
|
1f8d842706 | ||
|
|
cc3061d762 | ||
|
|
da2d50fbe2 | ||
|
|
9b9a1dfaed | ||
|
|
cfb02bd135 | ||
|
|
36fd68a527 | ||
|
|
e27ab22bd0 | ||
|
|
0776ee5b9f | ||
|
|
8b1bc0e1ac | ||
|
|
9715a9e018 | ||
|
|
62e441c8d7 | ||
|
|
665fd4c320 | ||
|
|
fdabd40f24 | ||
|
|
617dbff52b | ||
|
|
1740c3626a | ||
|
|
5b0e0b449c |
@@ -1445,6 +1445,7 @@ if (execStat) {
|
||||
if ("sftp".equals(type)) { %>
|
||||
((com.jcraft.jsch.ChannelSftp) obj_conn).quit();
|
||||
<%} else { %>
|
||||
((org.apache.commons.net.ftp.FTPClient) obj_conn).logout();
|
||||
((org.apache.commons.net.ftp.FTPClient) obj_conn).disconnect();
|
||||
<%}%>
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>checkArchive-1.1-20190917</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>checkArchive</artifactId>
|
||||
<version>1.2-20210901</version>
|
||||
<name>checkArchive</name>
|
||||
<description>Dependence for tFileArchive and tFileUnAchive</description>
|
||||
<url>http://maven.apache.org</url>
|
||||
@@ -10,7 +10,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
<java.source.version>1.7</java.source.version>
|
||||
<java.source.version>1.8</java.source.version>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
@@ -41,7 +41,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<version>1.19</version>
|
||||
<version>1.21</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -22,6 +22,7 @@ import javax.xml.bind.annotation.XmlType;
|
||||
* <element name="description" type="{http://www.w3.org/2001/XMLSchema}string" minOccurs="0"/>
|
||||
* <element name="name" type="{http://www.w3.org/2001/XMLSchema}string" minOccurs="0"/>
|
||||
* <element name="processSteps" type="{http://www.talend.com/mdm}WSTransformerProcessStep" maxOccurs="unbounded" minOccurs="0"/>
|
||||
* <element name="withAdminPermissions" type="{http://www.w3.org/2001/XMLSchema}boolean" minOccurs="0"/>
|
||||
* </sequence>
|
||||
* </restriction>
|
||||
* </complexContent>
|
||||
@@ -34,7 +35,8 @@ import javax.xml.bind.annotation.XmlType;
|
||||
@XmlType(name = "WSTransformerV2", propOrder = {
|
||||
"description",
|
||||
"name",
|
||||
"processSteps"
|
||||
"processSteps",
|
||||
"withAdminPermissions"
|
||||
})
|
||||
public class WSTransformerV2 {
|
||||
|
||||
@@ -42,6 +44,7 @@ public class WSTransformerV2 {
|
||||
protected String name;
|
||||
@XmlElement(nillable = true)
|
||||
protected List<WSTransformerProcessStep> processSteps;
|
||||
protected Boolean withAdminPermissions;
|
||||
|
||||
/**
|
||||
* Default no-arg constructor
|
||||
@@ -55,10 +58,11 @@ public class WSTransformerV2 {
|
||||
* Fully-initialising value constructor
|
||||
*
|
||||
*/
|
||||
public WSTransformerV2(final String description, final String name, final List<WSTransformerProcessStep> processSteps) {
|
||||
public WSTransformerV2(final String description, final String name, final List<WSTransformerProcessStep> processSteps, final Boolean withAdminPermissions) {
|
||||
this.description = description;
|
||||
this.name = name;
|
||||
this.processSteps = processSteps;
|
||||
this.withAdminPermissions = withAdminPermissions;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -138,4 +142,28 @@ public class WSTransformerV2 {
|
||||
return this.processSteps;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the value of the withAdminPermissions property.
|
||||
*
|
||||
* @return
|
||||
* possible object is
|
||||
* {@link Boolean }
|
||||
*
|
||||
*/
|
||||
public Boolean isWithAdminPermissions() {
|
||||
return withAdminPermissions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the value of the withAdminPermissions property.
|
||||
*
|
||||
* @param value
|
||||
* allowed object is
|
||||
* {@link Boolean }
|
||||
*
|
||||
*/
|
||||
public void setWithAdminPermissions(Boolean value) {
|
||||
this.withAdminPermissions = value;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend-codegen-utils</artifactId>
|
||||
<!-- release for revert version of library -->
|
||||
<version>0.30.0</version>
|
||||
<version>0.31.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
|
||||
@@ -69,13 +69,13 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-api</artifactId>
|
||||
<version>2.12.1</version>
|
||||
<version>2.17.1</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
<version>2.12.1</version>
|
||||
<version>2.17.1</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
@@ -97,4 +97,4 @@
|
||||
</pluginManagement>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
</project>
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>talend-orc</artifactId>
|
||||
<version>1.0-20211008</version>
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<hadoop.version>3.2.2</hadoop.version>
|
||||
<apache.orc.version>1.7.0</apache.orc.version>
|
||||
<junit.jupiter.version>5.7.2</junit.jupiter.version>
|
||||
<hamcrest.version>1.3</hamcrest.version>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
</properties>
|
||||
|
||||
<distributionManagement>
|
||||
<snapshotRepository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceSnapshot/</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
</snapshotRepository>
|
||||
<repository>
|
||||
<id>talend_nexus_deployment</id>
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.orc</groupId>
|
||||
<artifactId>orc-core</artifactId>
|
||||
<version>${apache.orc.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-server</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-servlet</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jersey-server</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jersey-servlet</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>log4j</artifactId>
|
||||
<groupId>log4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-webapp</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>javax.servlet-api</artifactId>
|
||||
<groupId>javax.servlet</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdfs-client</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-server</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-servlet</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jersey-server</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jersey-servlet</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>log4j</artifactId>
|
||||
<groupId>log4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-webapp</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>javax.servlet-api</artifactId>
|
||||
<groupId>javax.servlet</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<version>${junit.jupiter.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter-api</artifactId>
|
||||
<version>${junit.jupiter.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter-engine</artifactId>
|
||||
<version>${junit.jupiter.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter-params</artifactId>
|
||||
<version>${junit.jupiter.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.platform</groupId>
|
||||
<artifactId>junit-platform-launcher</artifactId>
|
||||
<version>1.7.2</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.hamcrest</groupId>
|
||||
<artifactId>hamcrest-library</artifactId>
|
||||
<version>${hamcrest.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.hamcrest</groupId>
|
||||
<artifactId>hamcrest-library</artifactId>
|
||||
<version>${hamcrest.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.8.0</version>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@@ -0,0 +1,24 @@
|
||||
package org.talend.orc;
|
||||
|
||||
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
|
||||
|
||||
abstract class ORCCommonUtils {
|
||||
/**
|
||||
* Check that the map type for the key is BYTES, LONG or DOUBLE and that the key
|
||||
* type is LONG, DOUBLE, BYTES, DECIMAL or TIMESTAMP.
|
||||
*
|
||||
* @param mapVector a MapColumnVector
|
||||
* @return true if the key and value types conform to the limits described
|
||||
* above.
|
||||
*/
|
||||
public static boolean checkMapColumnVectorTypes(MapColumnVector mapVector) {
|
||||
ColumnVector.Type keyType = mapVector.keys.type;
|
||||
ColumnVector.Type valueType = mapVector.values.type;
|
||||
return (keyType == ColumnVector.Type.BYTES || keyType == ColumnVector.Type.LONG
|
||||
|| keyType == ColumnVector.Type.DOUBLE)
|
||||
&& (valueType == ColumnVector.Type.LONG || valueType == ColumnVector.Type.DOUBLE
|
||||
|| valueType == ColumnVector.Type.BYTES || valueType == ColumnVector.Type.DECIMAL
|
||||
|| valueType == ColumnVector.Type.TIMESTAMP);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
package org.talend.orc;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang3.tuple.ImmutablePair;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
|
||||
import org.apache.orc.TypeDescription;
|
||||
|
||||
public class ORCReadUtils {
|
||||
public static Object readColumnByName(VectorizedRowBatch batch, String columnName, TypeDescription schema,
|
||||
int rowNum) {
|
||||
List<String> allColumnNames = schema.getFieldNames();
|
||||
int colIndex = allColumnNames.indexOf(columnName);
|
||||
if (colIndex < 0 || colIndex > batch.cols.length - 1) {
|
||||
return null;
|
||||
} else {
|
||||
org.apache.hadoop.hive.ql.exec.vector.ColumnVector colVector = batch.cols[colIndex];
|
||||
TypeDescription fieldType = schema.getChildren().get(colIndex);
|
||||
int colRow = colVector.isRepeating ? 0 : rowNum;
|
||||
Object value = readColumn(colVector, fieldType, colRow);
|
||||
return value;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static Object readColumn(ColumnVector colVec, TypeDescription colType, int rowNum) {
|
||||
Object columnObj = null;
|
||||
if (!colVec.isNull[rowNum]) {
|
||||
switch (colVec.type) {
|
||||
case BYTES:
|
||||
columnObj = readBytesVal(colVec, colType, rowNum);
|
||||
break;
|
||||
case DECIMAL:
|
||||
columnObj = readDecimalVal(colVec, rowNum);
|
||||
break;
|
||||
case DOUBLE:
|
||||
columnObj = readDoubleVal(colVec, colType, rowNum);
|
||||
break;
|
||||
case LIST:
|
||||
columnObj = readListVal(colVec, colType, rowNum);
|
||||
break;
|
||||
case LONG:
|
||||
columnObj = readLongVal(colVec, colType, rowNum);
|
||||
break;
|
||||
case MAP:
|
||||
columnObj = readMapVal(colVec, colType, rowNum);
|
||||
break;
|
||||
case STRUCT:
|
||||
columnObj = readStructVal(colVec, colType, rowNum);
|
||||
break;
|
||||
case TIMESTAMP:
|
||||
columnObj = readTimestampVal(colVec, colType, rowNum);
|
||||
break;
|
||||
case UNION:
|
||||
columnObj = readUnionVal(colVec, colType, rowNum);
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException("readColumn: unsupported ORC file column type: " + colVec.type.name());
|
||||
}
|
||||
}
|
||||
return columnObj;
|
||||
}
|
||||
|
||||
private static Object readListVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
|
||||
Object listValues = null;
|
||||
if (!colVec.isNull[rowNum]) {
|
||||
ListColumnVector listVector = (ListColumnVector) colVec;
|
||||
ColumnVector listChildVector = listVector.child;
|
||||
TypeDescription childType = colType.getChildren().get(0);
|
||||
switch (listChildVector.type) {
|
||||
case BYTES:
|
||||
listValues = readBytesListValues(listVector, childType, rowNum);
|
||||
break;
|
||||
case DECIMAL:
|
||||
listValues = readDecimalListValues(listVector, rowNum);
|
||||
break;
|
||||
case DOUBLE:
|
||||
listValues = readDoubleListValues(listVector, rowNum);
|
||||
break;
|
||||
case LONG:
|
||||
listValues = readLongListValues(listVector, childType, rowNum);
|
||||
break;
|
||||
case TIMESTAMP:
|
||||
listValues = readTimestampListValues(listVector, childType, rowNum);
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException(listVector.type.name() + " is not supported for ListColumnVectors");
|
||||
}
|
||||
}
|
||||
return listValues;
|
||||
}
|
||||
|
||||
private static List<Object> readLongListVector(LongColumnVector longVector, TypeDescription childType, int offset,
|
||||
int numValues) {
|
||||
List<Object> longList = new ArrayList<>();
|
||||
for (int i = 0; i < numValues; i++) {
|
||||
if (!longVector.isNull[offset + i]) {
|
||||
long longVal = longVector.vector[offset + i];
|
||||
if (childType.getCategory() == TypeDescription.Category.BOOLEAN) {
|
||||
Boolean boolVal = longVal == 0 ? Boolean.valueOf(false) : Boolean.valueOf(true);
|
||||
longList.add(boolVal);
|
||||
} else if (childType.getCategory() == TypeDescription.Category.INT) {
|
||||
Integer intObj = (int) longVal;
|
||||
longList.add(intObj);
|
||||
} else {
|
||||
longList.add(longVal);
|
||||
}
|
||||
} else {
|
||||
longList.add(null);
|
||||
}
|
||||
}
|
||||
return longList;
|
||||
}
|
||||
|
||||
private static Object readLongListValues(ListColumnVector listVector, TypeDescription childType, int rowNum) {
|
||||
int offset = (int) listVector.offsets[rowNum];
|
||||
int numValues = (int) listVector.lengths[rowNum];
|
||||
LongColumnVector longVector = (LongColumnVector) listVector.child;
|
||||
return readLongListVector(longVector, childType, offset, numValues);
|
||||
}
|
||||
|
||||
private static Object readTimestampListVector(TimestampColumnVector timestampVector, TypeDescription childType,
|
||||
int offset, int numValues) {
|
||||
List<Object> timestampList = new ArrayList<>();
|
||||
for (int i = 0; i < numValues; i++) {
|
||||
if (!timestampVector.isNull[offset + i]) {
|
||||
int nanos = timestampVector.nanos[offset + i];
|
||||
long millisec = timestampVector.time[offset + i];
|
||||
Timestamp timestamp = new Timestamp(millisec);
|
||||
timestamp.setNanos(nanos);
|
||||
if (childType.getCategory() == TypeDescription.Category.DATE) {
|
||||
Date date = new Date(timestamp.getTime());
|
||||
timestampList.add(date);
|
||||
} else {
|
||||
timestampList.add(timestamp);
|
||||
}
|
||||
} else {
|
||||
timestampList.add(null);
|
||||
}
|
||||
}
|
||||
return timestampList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read either Timestamp or Date values, depending on the definition in the
|
||||
* schema.
|
||||
*/
|
||||
private static Object readTimestampListValues(ListColumnVector listVector, TypeDescription childType, int rowNum) {
|
||||
int offset = (int) listVector.offsets[rowNum];
|
||||
int numValues = (int) listVector.lengths[rowNum];
|
||||
TimestampColumnVector timestampVec = (TimestampColumnVector) listVector.child;
|
||||
return readTimestampListVector(timestampVec, childType, offset, numValues);
|
||||
}
|
||||
|
||||
private static Object readDecimalListVector(DecimalColumnVector decimalVector, int offset, int numValues,
|
||||
int batchRowNum) {
|
||||
List<Object> decimalList = new ArrayList<>();
|
||||
for (int i = 0; i < numValues; i++) {
|
||||
if (!decimalVector.isNull[offset + i]) {
|
||||
BigDecimal bigDecimal = decimalVector.vector[batchRowNum].getHiveDecimal().bigDecimalValue();
|
||||
decimalList.add(bigDecimal);
|
||||
} else {
|
||||
decimalList.add(null);
|
||||
}
|
||||
}
|
||||
return decimalList;
|
||||
}
|
||||
|
||||
private static Object readDecimalListValues(ListColumnVector listVector, int rowNum) {
|
||||
int offset = (int) listVector.offsets[rowNum];
|
||||
int numValues = (int) listVector.lengths[rowNum];
|
||||
DecimalColumnVector decimalVec = (DecimalColumnVector) listVector.child;
|
||||
return readDecimalListVector(decimalVec, offset, numValues, rowNum);
|
||||
}
|
||||
|
||||
private static Object readBytesListVector(BytesColumnVector bytesVec, TypeDescription childType, int offset,
|
||||
int numValues) {
|
||||
List<Object> bytesValList = new ArrayList<>();
|
||||
for (int i = 0; i < numValues; i++) {
|
||||
if (!bytesVec.isNull[offset + i]) {
|
||||
byte[] byteArray = bytesVec.vector[offset + i];
|
||||
int vecLen = bytesVec.length[offset + i];
|
||||
int vecStart = bytesVec.start[offset + i];
|
||||
byte[] vecCopy = Arrays.copyOfRange(byteArray, vecStart, vecStart + vecLen);
|
||||
if (childType.getCategory() == TypeDescription.Category.STRING) {
|
||||
String str = new String(vecCopy);
|
||||
bytesValList.add(str);
|
||||
} else {
|
||||
bytesValList.add(vecCopy);
|
||||
}
|
||||
} else {
|
||||
bytesValList.add(null);
|
||||
}
|
||||
}
|
||||
return bytesValList;
|
||||
}
|
||||
|
||||
private static Object readBytesListValues(ListColumnVector listVector, TypeDescription childType, int rowNum) {
|
||||
int offset = (int) listVector.offsets[rowNum];
|
||||
int numValues = (int) listVector.lengths[rowNum];
|
||||
BytesColumnVector bytesVec = (BytesColumnVector) listVector.child;
|
||||
return readBytesListVector(bytesVec, childType, offset, numValues);
|
||||
}
|
||||
|
||||
private static Object readDoubleListVector(DoubleColumnVector doubleVec, int offset, int numValues) {
|
||||
List<Object> doubleList = new ArrayList<>();
|
||||
for (int i = 0; i < numValues; i++) {
|
||||
if (!doubleVec.isNull[offset + i]) {
|
||||
Double doubleVal = doubleVec.vector[offset + i];
|
||||
doubleList.add(doubleVal);
|
||||
} else {
|
||||
doubleList.add(null);
|
||||
}
|
||||
}
|
||||
return doubleList;
|
||||
}
|
||||
|
||||
private static Object readDoubleListValues(ListColumnVector listVector, int rowNum) {
|
||||
|
||||
int offset = (int) listVector.offsets[rowNum];
|
||||
int numValues = (int) listVector.lengths[rowNum];
|
||||
DoubleColumnVector doubleVec = (DoubleColumnVector) listVector.child;
|
||||
return readDoubleListVector(doubleVec, offset, numValues);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<Object> readMapVector(ColumnVector mapVector, TypeDescription childType, int offset,
|
||||
int numValues, int rowNum) {
|
||||
List<Object> mapList = null;
|
||||
switch (mapVector.type) {
|
||||
case BYTES:
|
||||
mapList = (List<Object>) readBytesListVector((BytesColumnVector) mapVector, childType, offset, numValues);
|
||||
break;
|
||||
case DECIMAL:
|
||||
mapList = (List<Object>) readDecimalListVector((DecimalColumnVector) mapVector, offset, numValues, rowNum);
|
||||
;
|
||||
break;
|
||||
case DOUBLE:
|
||||
mapList = (List<Object>) readDoubleListVector((DoubleColumnVector) mapVector, offset, numValues);
|
||||
break;
|
||||
case LONG:
|
||||
mapList = readLongListVector((LongColumnVector) mapVector, childType, offset, numValues);
|
||||
break;
|
||||
case TIMESTAMP:
|
||||
mapList = (List<Object>) readTimestampListVector((TimestampColumnVector) mapVector, childType, offset,
|
||||
numValues);
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException(mapVector.type.name() + " is not supported for MapColumnVectors");
|
||||
}
|
||||
return mapList;
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Read a Map column value (e.g., a set of keys and their associated values).
|
||||
* </p>
|
||||
* <p>
|
||||
* The Map key and value types are the first and second children in the children
|
||||
* TypeDescription List. From the TypeDescription source:
|
||||
* </p>
|
||||
*
|
||||
* <pre>
|
||||
* result.children.add(keyType);
|
||||
* result.children.add(valueType);
|
||||
* </pre>
|
||||
*/
|
||||
private static Object readMapVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
|
||||
Map<Object, Object> objMap = new HashMap<>();
|
||||
MapColumnVector mapVector = (MapColumnVector) colVec;
|
||||
if (ORCCommonUtils.checkMapColumnVectorTypes(mapVector)) {
|
||||
int mapSize = (int) mapVector.lengths[rowNum];
|
||||
int offset = (int) mapVector.offsets[rowNum];
|
||||
List<TypeDescription> mapTypes = colType.getChildren();
|
||||
TypeDescription keyType = mapTypes.get(0);
|
||||
TypeDescription valueType = mapTypes.get(1);
|
||||
ColumnVector keyChild = mapVector.keys;
|
||||
ColumnVector valueChild = mapVector.values;
|
||||
List<Object> keyList = readMapVector(keyChild, keyType, offset, mapSize, rowNum);
|
||||
List<Object> valueList = readMapVector(valueChild, valueType, offset, mapSize, rowNum);
|
||||
assert (keyList.size() == valueList.size());
|
||||
for (int i = 0; i < keyList.size(); i++) {
|
||||
objMap.put(keyList.get(i), valueList.get(i));
|
||||
}
|
||||
} else {
|
||||
throw new RuntimeException("readMapVal: unsupported key or value types");
|
||||
}
|
||||
return objMap;
|
||||
}
|
||||
|
||||
private static Object readUnionVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
|
||||
Pair<TypeDescription, Object> columnValuePair;
|
||||
UnionColumnVector unionVector = (UnionColumnVector) colVec;
|
||||
int tagVal = unionVector.tags[rowNum];
|
||||
List<TypeDescription> unionFieldTypes = colType.getChildren();
|
||||
if (tagVal < unionFieldTypes.size()) {
|
||||
TypeDescription fieldType = unionFieldTypes.get(tagVal);
|
||||
if (tagVal < unionVector.fields.length) {
|
||||
ColumnVector fieldVector = unionVector.fields[tagVal];
|
||||
int colRow = fieldVector.isRepeating ? 0 : rowNum;
|
||||
Object unionValue = readColumn(fieldVector, fieldType, colRow);
|
||||
columnValuePair = new ImmutablePair<>(fieldType, unionValue);
|
||||
} else {
|
||||
throw new RuntimeException("readUnionVal: union tag value out of range for union column vectors");
|
||||
}
|
||||
} else {
|
||||
throw new RuntimeException("readUnionVal: union tag value out of range for union types");
|
||||
}
|
||||
return columnValuePair;
|
||||
}
|
||||
|
||||
private static Object readStructVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
|
||||
Object structObj = null;
|
||||
if (!colVec.isNull[rowNum]) {
|
||||
List<Object> fieldValList = new ArrayList<>();
|
||||
StructColumnVector structVector = (StructColumnVector) colVec;
|
||||
ColumnVector[] fieldVec = structVector.fields;
|
||||
List<TypeDescription> fieldTypes = colType.getChildren();
|
||||
assert (fieldVec.length == fieldTypes.size());
|
||||
for (int i = 0; i < fieldVec.length; i++) {
|
||||
int colRow = fieldVec[i].isRepeating ? 0 : rowNum;
|
||||
Object fieldObj = readColumn(fieldVec[i], fieldTypes.get(i), colRow);
|
||||
fieldValList.add(fieldObj);
|
||||
}
|
||||
structObj = fieldValList;
|
||||
}
|
||||
return structObj;
|
||||
}
|
||||
|
||||
private static Object readTimestampVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
|
||||
Object timestampVal = null;
|
||||
if (!colVec.isNull[rowNum]) {
|
||||
TimestampColumnVector timestampVec = (TimestampColumnVector) colVec;
|
||||
int nanos = timestampVec.nanos[rowNum];
|
||||
long millisec = timestampVec.time[rowNum];
|
||||
Timestamp timestamp = new Timestamp(millisec);
|
||||
timestamp.setNanos(nanos);
|
||||
timestampVal = timestamp;
|
||||
if (colType.getCategory() == TypeDescription.Category.DATE) {
|
||||
timestampVal = new Date(timestamp.getTime());
|
||||
}
|
||||
}
|
||||
return timestampVal;
|
||||
}
|
||||
|
||||
private static Object readDecimalVal(ColumnVector colVec, int rowNum) {
|
||||
Object decimalObj = null;
|
||||
if (!colVec.isNull[rowNum]) {
|
||||
DecimalColumnVector decimalVec = (DecimalColumnVector) colVec;
|
||||
decimalObj = decimalVec.vector[rowNum].getHiveDecimal().bigDecimalValue().setScale(decimalVec.scale);
|
||||
}
|
||||
return decimalObj;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a Long or Boolean value
|
||||
*
|
||||
* @param colVec the column vector
|
||||
* @param colType the type of the column
|
||||
* @param rowNum the ORC file row number.
|
||||
* @return a Boolean or Long object
|
||||
*/
|
||||
private static Object readLongVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
|
||||
Object colObj = null;
|
||||
if (!colVec.isNull[rowNum]) {
|
||||
LongColumnVector longVec = (LongColumnVector) colVec;
|
||||
Long longVal = longVec.vector[rowNum];
|
||||
colObj = longVal;
|
||||
if (colType.getCategory() == TypeDescription.Category.INT) {
|
||||
colObj = longVal.intValue();
|
||||
} else if (colType.getCategory() == TypeDescription.Category.BOOLEAN) {
|
||||
colObj = longVal == 1 ? Boolean.TRUE : Boolean.FALSE;
|
||||
} else if (colType.getCategory() == TypeDescription.Category.DATE) {
|
||||
colObj = new Date(longVal * 86400000);
|
||||
} else if (colType.getCategory() == TypeDescription.Category.BYTE) {
|
||||
colObj = longVal.byteValue();
|
||||
} else if (colType.getCategory() == TypeDescription.Category.SHORT) {
|
||||
colObj = longVal.shortValue();
|
||||
}
|
||||
}
|
||||
return colObj;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a Double or Float value
|
||||
*
|
||||
* @param colVec the column vector
|
||||
* @param colType the type of the column
|
||||
* @param rowNum the ORC file row number.
|
||||
* @return a Double or Float object
|
||||
*/
|
||||
private static Object readDoubleVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
|
||||
Object colObj = null;
|
||||
if (!colVec.isNull[rowNum]) {
|
||||
DoubleColumnVector longVec = (DoubleColumnVector) colVec;
|
||||
Double doubleVal = longVec.vector[rowNum];
|
||||
colObj = doubleVal;
|
||||
if (colType.getCategory() == TypeDescription.Category.FLOAT) {
|
||||
colObj = doubleVal.floatValue();
|
||||
}
|
||||
}
|
||||
return colObj;
|
||||
}
|
||||
|
||||
private static Object readBytesVal(ColumnVector colVec, TypeDescription colType, int rowNum) {
|
||||
Object bytesObj = null;
|
||||
if (!colVec.isNull[rowNum]) {
|
||||
BytesColumnVector bytesVector = (BytesColumnVector) colVec;
|
||||
byte[] columnBytes = bytesVector.vector[rowNum];
|
||||
int vecLen = bytesVector.length[rowNum];
|
||||
int vecStart = bytesVector.start[rowNum];
|
||||
byte[] vecCopy = Arrays.copyOfRange(columnBytes, vecStart, vecStart + vecLen);
|
||||
if (colType.getCategory() == TypeDescription.Category.STRING ||colType.getCategory() == TypeDescription.Category.VARCHAR) {
|
||||
bytesObj = new String(vecCopy);
|
||||
} else if (colType.getCategory() == TypeDescription.Category.CHAR) {
|
||||
String charStr = new String(vecCopy);
|
||||
bytesObj = charStr;
|
||||
} else {
|
||||
bytesObj = vecCopy;
|
||||
}
|
||||
}
|
||||
return bytesObj;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,884 @@
|
||||
package org.talend.orc;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.LocalDate;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TimeZone;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.apache.hadoop.hive.common.type.HiveDecimal;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.DateColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.MultiValuedColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
|
||||
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
|
||||
import org.apache.orc.TypeDescription;
|
||||
|
||||
public class ORCWriteUtils {
|
||||
private static final Predicate<Object> isInteger = Integer.class::isInstance;
|
||||
private static final Predicate<Object> isLong = Long.class::isInstance;
|
||||
private static final Predicate<Object> isDouble = Double.class::isInstance;
|
||||
private static final Predicate<Object> isString = String.class::isInstance;
|
||||
private static final Predicate<Object> isBigDecimal = BigDecimal.class::isInstance;
|
||||
private static final Predicate<Object> isDate = Date.class::isInstance;
|
||||
|
||||
public static void setColumnByName(VectorizedRowBatch batch, String columnName, TypeDescription schema,
|
||||
Object colVal, int rowNum) {
|
||||
List<String> allColumnNames = schema.getFieldNames();
|
||||
int colIndex = allColumnNames.indexOf(columnName);
|
||||
if (colIndex < 0 || colIndex > batch.cols.length - 1) {
|
||||
return;
|
||||
} else {
|
||||
org.apache.hadoop.hive.ql.exec.vector.ColumnVector colVector = batch.cols[colIndex];
|
||||
TypeDescription fieldType = schema.getChildren().get(colIndex);
|
||||
setColumn(colVal, fieldType, columnName, colVector, rowNum);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a column value that is a String or a byte[] array.
|
||||
*
|
||||
* @param colVal the column value object
|
||||
* @param fieldName the name of the field (for error reporting)
|
||||
* @param bytesColVector the BytesColumnVector that the byte array will be added
|
||||
* to.
|
||||
* @param rowNum the ORC file row number
|
||||
*/
|
||||
private static void setByteColumnVector(Object colVal, String fieldName, BytesColumnVector bytesColVector,
|
||||
int rowNum) {
|
||||
if (colVal instanceof byte[] || colVal instanceof String || colVal instanceof Character) {
|
||||
byte[] byteVec;
|
||||
if (colVal instanceof String) {
|
||||
String strVal = (String) colVal;
|
||||
byteVec = strVal.getBytes(StandardCharsets.UTF_8);
|
||||
} else if (colVal instanceof Character) {
|
||||
String strVal = String.valueOf((char) colVal);
|
||||
byteVec = strVal.getBytes(StandardCharsets.UTF_8);
|
||||
} else {
|
||||
byteVec = (byte[]) colVal;
|
||||
}
|
||||
bytesColVector.setRef(rowNum, byteVec, 0, byteVec.length);
|
||||
} else {
|
||||
throw new RuntimeException(orcExceptionMsg("byte[] or String type expected for field ", fieldName, rowNum));
|
||||
}
|
||||
}
|
||||
|
||||
private static void setDecimalVector(Object colVal, String fieldName, DecimalColumnVector decimalColVector,
|
||||
int rowNum) {
|
||||
if (colVal instanceof BigDecimal) {
|
||||
BigDecimal bigDecimal = (BigDecimal) colVal;
|
||||
decimalColVector.precision = (short) bigDecimal.precision();
|
||||
decimalColVector.scale = (short) bigDecimal.scale();
|
||||
HiveDecimal hiveDecimal = HiveDecimal.create(bigDecimal);
|
||||
HiveDecimalWritable writeableDecimal = new HiveDecimalWritable(hiveDecimal);
|
||||
decimalColVector.vector[rowNum] = writeableDecimal;
|
||||
} else {
|
||||
throw new RuntimeException(orcExceptionMsg("BigDecimal type expected for field ", fieldName, rowNum));
|
||||
}
|
||||
}
|
||||
|
||||
private static void setDoubleVector(Object colVal, String fieldName, DoubleColumnVector doubleVector, int rowNum) {
|
||||
if (colVal instanceof Double) {
|
||||
doubleVector.vector[rowNum] = (Double) colVal;
|
||||
} else if (colVal instanceof Float) {
|
||||
Float fltVal = (Float) colVal;
|
||||
doubleVector.vector[rowNum] = fltVal.doubleValue();
|
||||
} else if (colVal instanceof BigDecimal) {
|
||||
doubleVector.vector[rowNum] = ((BigDecimal) colVal).doubleValue();
|
||||
} else {
|
||||
throw new RuntimeException(orcExceptionMsg("Double/Float/BigDecimal type expected for field ", fieldName, rowNum));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize a LongColumnVector value.
|
||||
*
|
||||
* @param colVal an object of type Boolean, Integer, Long or BigInteger.
|
||||
* @param fieldName the field name in the schema
|
||||
* @param longVector the LongColumnVector
|
||||
* @param rowNum the row number
|
||||
*/
|
||||
private static void setLongColumnVector(Object colVal, String fieldName, LongColumnVector longVector, int rowNum) {
|
||||
if (colVal instanceof Boolean) {
|
||||
Boolean bool = (Boolean) colVal;
|
||||
longVector.vector[rowNum] = (bool.equals(Boolean.TRUE)) ? Long.valueOf(1) : Long.valueOf(0);
|
||||
} else if (colVal instanceof Byte) {
|
||||
longVector.vector[rowNum] = (Byte) colVal;
|
||||
} else if (colVal instanceof Short) {
|
||||
longVector.vector[rowNum] = (Short) colVal;
|
||||
} else if (colVal instanceof Integer) {
|
||||
longVector.vector[rowNum] = (Integer) colVal;
|
||||
} else if (colVal instanceof Long) {
|
||||
longVector.vector[rowNum] = (Long) colVal;
|
||||
} else if (colVal instanceof BigInteger) {
|
||||
BigInteger bigInt = (BigInteger) colVal;
|
||||
longVector.vector[rowNum] = bigInt.longValue();
|
||||
} else {
|
||||
throw new RuntimeException(orcExceptionMsg("Long or Integer type expected for field ", fieldName, rowNum));
|
||||
}
|
||||
}
|
||||
|
||||
private static void setDateColumnVector(Object colVal, String fieldName, DateColumnVector dateVector, int rowNum) {
|
||||
if (colVal instanceof Date) {
|
||||
Date dateVal = (Date) colVal;
|
||||
Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
|
||||
cal.setTime(dateVal);
|
||||
long epochDay = LocalDate.of(cal.get(Calendar.YEAR), cal.get(Calendar.MONTH)+1, cal.get(Calendar.DAY_OF_MONTH)).toEpochDay();
|
||||
dateVector.vector[rowNum] = epochDay;
|
||||
} else {
|
||||
throw new RuntimeException(orcExceptionMsg("Date type expected for field ", fieldName, rowNum));
|
||||
}
|
||||
}
|
||||
|
||||
private static void setTimestampVector(Object colVal, String fieldName, TimestampColumnVector timestampVector,
|
||||
int rowNum) {
|
||||
if (colVal instanceof Timestamp) {
|
||||
timestampVector.set(rowNum, (Timestamp) colVal);
|
||||
} else if (colVal instanceof Date) {
|
||||
Date date = (Date) colVal;
|
||||
Timestamp ts = new Timestamp(date.getTime());
|
||||
|
||||
timestampVector.set(rowNum, ts);
|
||||
} else {
|
||||
throw new RuntimeException(
|
||||
orcExceptionMsg("Date or Timestamp type expected for field ", fieldName, rowNum));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* A union column can contain column vectors of more than one type. In the
|
||||
* TypeDescription createUnion() is called to create a TypeDescription for a
|
||||
* union column. The union values are added by calling the addUnionChild()
|
||||
* method on this TypeDescription object.
|
||||
* </p>
|
||||
* <p>
|
||||
* The class fields in the UnionColumnVector are shown below:
|
||||
* </p>
|
||||
*
|
||||
* <pre>
|
||||
* public class UnionColumnVector extends ColumnVector {
|
||||
* public int[] tags;
|
||||
* public ColumnVector[] fields;
|
||||
* </pre>
|
||||
* <p>
|
||||
* A tag value (
|
||||
*
|
||||
* <pre>
|
||||
* tags[rowNum]
|
||||
* </pre>
|
||||
*
|
||||
* ) is associated with each field value (
|
||||
*
|
||||
* <pre>
|
||||
* fields[rowNum])
|
||||
* </pre>
|
||||
*
|
||||
* . The tag value serves as an index for the field type. For example, if there
|
||||
* are three field types defined:
|
||||
* <ol>
|
||||
* <li>Long</li>
|
||||
* <li>Double</li>
|
||||
* <li>String</li>
|
||||
* </ol>
|
||||
* The tag will have a value in the range of [0..2]
|
||||
* </p>
|
||||
* <p>
|
||||
* The tag value is needed to initialize the ColumnVector since without the tag
|
||||
* there is no way to know which union child should be initialized.
|
||||
* </p>
|
||||
*
|
||||
* @param colVal a Pair<ColumnVector.Type, Object> object with the
|
||||
* union type and the object that will be used to initialize
|
||||
* the union child ColumnVector.
|
||||
* @param fieldName The name of the union field
|
||||
* @param unionVector The UnionColumnVector to be initialized
|
||||
* @param rowNum the ORC file row number.
|
||||
*/
|
||||
private static void setUnionColumnVector(Object colVal, TypeDescription unionTypeDesc, String fieldName,
|
||||
UnionColumnVector unionVector, int rowNum) {
|
||||
@SuppressWarnings("unchecked")
|
||||
Pair<TypeDescription, Object> unionValuePair = (Pair<TypeDescription, Object>) colVal;
|
||||
TypeDescription unionValType = unionValuePair.getLeft();
|
||||
List<TypeDescription> unionChildrenTypes = unionTypeDesc.getChildren();
|
||||
Object unionColVal = unionValuePair.getRight();
|
||||
boolean found = false;
|
||||
for (int i = 0; i < unionChildrenTypes.size(); i++) {
|
||||
if (unionChildrenTypes.get(i).getCategory() == unionValType.getCategory()) {
|
||||
unionVector.tags[rowNum] = i;
|
||||
ColumnVector unionFieldVec = unionVector.fields[i];
|
||||
setColumn(unionColVal, unionChildrenTypes.get(i), fieldName, unionFieldVec, rowNum);
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
throw new RuntimeException("writeUnionColumnVector: Bad type enumeration "
|
||||
+ unionValType.getCategory().getName() + " passed for field " + fieldName);
|
||||
}
|
||||
}
|
||||
|
||||
private static void setLongListVector(List<Object> longValList, LongColumnVector longVector, int offset,
|
||||
String fieldName) {
|
||||
for (int i = 0; i < longValList.size(); i++) {
|
||||
Object objVal = longValList.get(i);
|
||||
if (objVal != null) {
|
||||
if (objVal instanceof Integer) {
|
||||
longVector.vector[offset + i] = (Integer) objVal;
|
||||
} else if (objVal instanceof Long) {
|
||||
longVector.vector[offset + i] = (Long) objVal;
|
||||
} else {
|
||||
throw new RuntimeException("List<Integer> expected for field " + fieldName);
|
||||
}
|
||||
} else {
|
||||
longVector.isNull[offset + i] = true;
|
||||
longVector.noNulls = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void setLongList(List<Object> colValList, ListColumnVector listVector, String fieldName,
|
||||
int rowNum) {
|
||||
LongColumnVector longVector = (LongColumnVector) listVector.child;
|
||||
int offset = (int) listVector.offsets[rowNum];
|
||||
setLongListVector(colValList, longVector, offset, fieldName);
|
||||
}
|
||||
|
||||
private static void setDoubleListVector(List<Object> doubleValList, DoubleColumnVector doubleVector, int offset,
|
||||
String fieldName) {
|
||||
for (int i = 0; i < doubleValList.size(); i++) {
|
||||
Object objVal = doubleValList.get(i);
|
||||
if (objVal != null) {
|
||||
if (objVal instanceof Double) {
|
||||
doubleVector.vector[offset + i] = (Double) objVal;
|
||||
} else if (objVal instanceof Float) {
|
||||
Float fltVal = (Float) objVal;
|
||||
doubleVector.vector[offset + i] = fltVal.doubleValue();
|
||||
} else {
|
||||
throw new RuntimeException("List<Double> expected for field " + fieldName);
|
||||
}
|
||||
} else {
|
||||
doubleVector.isNull[offset + i] = true;
|
||||
doubleVector.noNulls = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void setDoubleList(List<Object> doubleValList, ListColumnVector listVector, String fieldName,
|
||||
int rowNum) {
|
||||
DoubleColumnVector vecChild = (DoubleColumnVector) listVector.child;
|
||||
int offset = (int) listVector.offsets[rowNum];
|
||||
setDoubleListVector(doubleValList, vecChild, offset, fieldName);
|
||||
}
|
||||
|
||||
private static void setTimestampListVector(List<Object> valueList, TimestampColumnVector timestampVector,
|
||||
int offset, String fieldName) {
|
||||
for (int i = 0; i < valueList.size(); i++) {
|
||||
Object objVal = valueList.get(i);
|
||||
if (objVal != null) {
|
||||
if (objVal instanceof Date) {
|
||||
Timestamp ts = (objVal instanceof Timestamp) ? (Timestamp) objVal
|
||||
: new Timestamp(((Date) objVal).getTime());
|
||||
timestampVector.time[offset + i] = ts.getTime();
|
||||
timestampVector.nanos[offset + i] = ts.getNanos();
|
||||
} else {
|
||||
throw new RuntimeException("List<Date> or List<Timestamp> expected for field " + fieldName);
|
||||
}
|
||||
} else {
|
||||
timestampVector.isNull[offset + i] = true;
|
||||
timestampVector.noNulls = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the vector values for a ListColumnVector of Date or Timestamp
|
||||
* values.
|
||||
*
|
||||
* @param colValList a list of Timestamp or java.util.Date objects
|
||||
* @param listVector A ListColumnVector with a child that will contain the
|
||||
* vector values.
|
||||
* @param fieldName The field name in the schema for this ORC element
|
||||
* @param rowNum The ORC file row number
|
||||
*/
|
||||
private static void setTimestampList(List<Object> colValList, ListColumnVector listVector, String fieldName,
|
||||
int rowNum) {
|
||||
TimestampColumnVector timestampVector = (TimestampColumnVector) listVector.child;
|
||||
int offset = (int) listVector.offsets[rowNum];
|
||||
setTimestampListVector(colValList, timestampVector, offset, fieldName);
|
||||
}
|
||||
|
||||
private static void setDecimalListVector(List<Object> decimalValList, DecimalColumnVector decimalVector, int offset,
|
||||
String fieldName) {
|
||||
for (int i = 0; i < decimalValList.size(); i++) {
|
||||
Object objVal = decimalValList.get(i);
|
||||
if (objVal != null) {
|
||||
if (objVal instanceof BigDecimal) {
|
||||
BigDecimal bigDecimal = (BigDecimal) objVal;
|
||||
decimalVector.precision = (short) bigDecimal.precision();
|
||||
decimalVector.scale = (short) bigDecimal.scale();
|
||||
HiveDecimal hiveDecimal = HiveDecimal.create(bigDecimal);
|
||||
HiveDecimalWritable writeableDecimal = new HiveDecimalWritable(hiveDecimal);
|
||||
decimalVector.vector[offset + i] = writeableDecimal;
|
||||
} else {
|
||||
throw new RuntimeException("BigDecimal value expected for field " + fieldName);
|
||||
}
|
||||
} else {
|
||||
decimalVector.isNull[offset + i] = true;
|
||||
decimalVector.noNulls = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param colValList a list of BigDecimal values to initialize the
|
||||
* ListColumnVector child
|
||||
* @param listVector the ListColumnVector with the DecimalColumnVector child
|
||||
* @param fieldName the field name for the ListColumnVector/DecimalColumnVector
|
||||
* column
|
||||
* @param rowNum the ORC file row number
|
||||
*/
|
||||
private static void setDecimalList(List<Object> colValList, ListColumnVector listVector, String fieldName,
|
||||
int rowNum) {
|
||||
DecimalColumnVector decimalVector = (DecimalColumnVector) listVector.child;
|
||||
int offset = (int) listVector.offsets[rowNum];
|
||||
setDecimalListVector(colValList, decimalVector, offset, fieldName);
|
||||
}
|
||||
|
||||
private static void setBytesListVector(List<Object> valueList, BytesColumnVector bytesVector, int offset,
|
||||
String fieldName) {
|
||||
for (int i = 0; i < valueList.size(); i++) {
|
||||
Object objVal = valueList.get(i);
|
||||
if (objVal != null) {
|
||||
if (objVal instanceof byte[] || objVal instanceof String) {
|
||||
byte[] byteVec = (objVal instanceof byte[]) ? (byte[]) objVal
|
||||
: ((String) objVal).getBytes(StandardCharsets.UTF_8);
|
||||
bytesVector.vector[offset + i] = byteVec;
|
||||
bytesVector.length[offset + i] = byteVec.length;
|
||||
} else {
|
||||
throw new RuntimeException("String or byte[] value expected for field " + fieldName);
|
||||
}
|
||||
} else {
|
||||
bytesVector.isNull[offset + i] = true;
|
||||
bytesVector.length[offset + i] = 0;
|
||||
bytesVector.noNulls = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize a ListColumnVector with a BytesColumnVector child with byte[]
|
||||
* values.
|
||||
*
|
||||
* @param colValList a list of byte[] or String values
|
||||
* @param listVector the parent ListColumnVector
|
||||
* @param fieldName the field name for the ORC column that contains the
|
||||
* ListColumnVector
|
||||
* @param rowNum the ORC file row number
|
||||
*/
|
||||
private static void setBytesList(List<Object> colValList, ListColumnVector listVector, String fieldName,
|
||||
int rowNum) {
|
||||
BytesColumnVector bytesVector = (BytesColumnVector) listVector.child;
|
||||
int offset = (int) listVector.offsets[rowNum];
|
||||
setBytesListVector(colValList, bytesVector, offset, fieldName);
|
||||
}
|
||||
|
||||
private static void setMultiValuedVectorParameters(MultiValuedColumnVector multiVector, int vecLength, int rowNum) {
|
||||
multiVector.lengths[rowNum] = vecLength;
|
||||
if (rowNum > 0) {
|
||||
multiVector.offsets[rowNum] = multiVector.lengths[rowNum - 1] + multiVector.offsets[rowNum - 1];
|
||||
}
|
||||
}
|
||||
|
||||
private static void setListVectorParameters(ListColumnVector listVec, int maxBatchSize, int vecLength, int rowNum) {
|
||||
setMultiValuedVectorParameters(listVec, vecLength, rowNum);
|
||||
listVec.child.ensureSize(maxBatchSize * vecLength, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize a ListColumnVector. The child of the vector is limited to the
|
||||
* scalar types long, double, String (or byte[])), BigDecimal or Date (or
|
||||
* Timestamp).
|
||||
*
|
||||
* @param colVal a List<Object>
|
||||
* @param typeDesc the schema definition for this column
|
||||
* @param fieldName the column field name
|
||||
* @param listVector the ListColumnVector parent of the vector type child
|
||||
* @param rowNum the ORC file row number.
|
||||
*/
|
||||
private static void setListColumnVector(Object colVal, TypeDescription typeDesc, String fieldName,
|
||||
ListColumnVector listVector, int rowNum) {
|
||||
if (colVal instanceof List) {
|
||||
@SuppressWarnings("unchecked")
|
||||
List<Object> objValList = (List<Object>) colVal;
|
||||
final int maxBatchSize = typeDesc.createRowBatch().getMaxSize();
|
||||
setListVectorParameters(listVector, maxBatchSize, objValList.size(), rowNum);
|
||||
ColumnVector.Type childType = listVector.child.type;
|
||||
switch (childType) {
|
||||
case LONG:
|
||||
setLongList(objValList, listVector, fieldName, rowNum);
|
||||
break;
|
||||
case DOUBLE:
|
||||
setDoubleList(objValList, listVector, fieldName, rowNum);
|
||||
break;
|
||||
case BYTES:
|
||||
setBytesList(objValList, listVector, fieldName, rowNum);
|
||||
break;
|
||||
case DECIMAL:
|
||||
setDecimalList(objValList, listVector, fieldName, rowNum);
|
||||
break;
|
||||
case TIMESTAMP:
|
||||
setTimestampList(objValList, listVector, fieldName, rowNum);
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException(childType.name() + " is not supported for ListColumnVector columns");
|
||||
}
|
||||
} else {
|
||||
throw new RuntimeException("List value expected for field " + fieldName);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that all elements in an Object list are of a particular type
|
||||
*
|
||||
* @param objList the Object list that is tested
|
||||
* @param typeTest a function that compares against a particular Object type
|
||||
* @return true if all elements are of the test type, false if one or more
|
||||
* elements are not of that type.
|
||||
*/
|
||||
private static boolean isListType(List<Object> objList, Predicate<Object> typeTest) {
|
||||
return !objList.stream().map(typeTest::test).collect(Collectors.toList()).contains(false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize a ColumnVector with Long values.
|
||||
*
|
||||
* @param valueList a list of Long values
|
||||
* @param colVector the LongColumnVector that will be initialized with the Long
|
||||
* values
|
||||
* @param offset the offset[rownum] value for the array
|
||||
* @param fieldName the field name for the Map column
|
||||
*/
|
||||
private static void setLongMapValues(List<Object> valueList, ColumnVector colVector, int offset, String fieldName) {
|
||||
if (isListType(valueList, isLong) || isListType(valueList, isInteger)) {
|
||||
LongColumnVector longVector = (LongColumnVector) colVector;
|
||||
setLongListVector(valueList, longVector, offset, fieldName);
|
||||
} else {
|
||||
throw new RuntimeException("For field " + fieldName + " Long values expected");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize a ColumnVector with Double values.
|
||||
*
|
||||
* @param valueList a list of Double values
|
||||
* @param colVector the DoubleColumnVector that will be initialized with the
|
||||
* Double values
|
||||
* @param offset the offset[rownum] value for the array
|
||||
* @param fieldName the field name for the Map column
|
||||
*/
|
||||
private static void setDoubleMapValues(List<Object> valueList, ColumnVector colVector, int offset,
|
||||
String fieldName) {
|
||||
if (isListType(valueList, isDouble)) {
|
||||
DoubleColumnVector doubleVector = (DoubleColumnVector) colVector;
|
||||
setDoubleListVector(valueList, doubleVector, offset, fieldName);
|
||||
} else {
|
||||
throw new RuntimeException("For field " + fieldName + " Double values expected");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize a ColumnVector with String values.
|
||||
*
|
||||
* @param valueList a list of String values
|
||||
* @param colVector the BytesColumnVector that will be initialized with the
|
||||
* String values
|
||||
* @param offset the offset[rownum] value for the array
|
||||
* @param fieldName the field name for the Map column
|
||||
*/
|
||||
private static void setStringMapValues(List<Object> valueList, ColumnVector colVector, int offset,
|
||||
String fieldName) {
|
||||
if (isListType(valueList, isString)) {
|
||||
BytesColumnVector doubleVector = (BytesColumnVector) colVector;
|
||||
setBytesListVector(valueList, doubleVector, offset, fieldName);
|
||||
} else {
|
||||
throw new RuntimeException("For field " + fieldName + " String values expected");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize a ColumnVector with BigDeciml values.
|
||||
*
|
||||
* @param valueList a list of BigDecimal
|
||||
* @param colVector the DecimalColumnVector that will be initialized with the
|
||||
* BigDecimal values
|
||||
* @param offset the offset[rownum] value for the array
|
||||
* @param fieldName the field name for the Map column
|
||||
*/
|
||||
private static void setDecimalMapValues(List<Object> valueList, ColumnVector colVector, int offset,
|
||||
String fieldName) {
|
||||
if (isListType(valueList, isBigDecimal)) {
|
||||
DecimalColumnVector decimalVector = (DecimalColumnVector) colVector;
|
||||
setDecimalListVector(valueList, decimalVector, offset, fieldName);
|
||||
} else {
|
||||
throw new RuntimeException("For field " + fieldName + " BigDecimal values expected");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize a ColumnVector with timestamp values.
|
||||
*
|
||||
* @param valueList a list of Date (or Timestamp) objects
|
||||
* @param colVector the TimestampColumnVector that will be initialized with the
|
||||
* Timestamp values
|
||||
* @param offset the offset[rownum] value for the array
|
||||
* @param fieldName the field name for the Map column
|
||||
*/
|
||||
private static void setTimestampMapValues(List<Object> valueList, ColumnVector colVector, int offset,
|
||||
String fieldName) {
|
||||
if (isListType(valueList, isDate)) {
|
||||
TimestampColumnVector timestampVector = (TimestampColumnVector) colVector;
|
||||
setTimestampListVector(valueList, timestampVector, offset, fieldName);
|
||||
} else {
|
||||
throw new RuntimeException("For field " + fieldName + " Date or Timestamp values expected");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the MapColumn value array vector. The type for this vector is limited to
|
||||
* long, double, bytes (String), Decimal and Timestamp.
|
||||
*
|
||||
* @param valueList a list of Objects to initialize the Map column value array.
|
||||
* @param colVector the column array vector to be initialized with the map
|
||||
* values.
|
||||
* @param offset the offset[rowNum] from the parent MapColumnVector
|
||||
* @param fieldName the name of the field for the MapColumnVector.
|
||||
*/
|
||||
private static void setMapValueVector(List<Object> valueList, ColumnVector colVector, int offset,
|
||||
String fieldName) {
|
||||
switch (colVector.type) {
|
||||
case LONG:
|
||||
setLongMapValues(valueList, colVector, offset, fieldName);
|
||||
break;
|
||||
case DOUBLE:
|
||||
setDoubleMapValues(valueList, colVector, offset, fieldName);
|
||||
break;
|
||||
case BYTES:
|
||||
setStringMapValues(valueList, colVector, offset, fieldName);
|
||||
break;
|
||||
case DECIMAL:
|
||||
setDecimalMapValues(valueList, colVector, offset, fieldName);
|
||||
break;
|
||||
case TIMESTAMP:
|
||||
setTimestampMapValues(valueList, colVector, offset, fieldName);
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException(
|
||||
"For field " + fieldName + " values must be long, double, String, BigDecimal or Timestamp");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Initialize a MapColumnVector with Long key values.
|
||||
* </p>
|
||||
*
|
||||
* @param mapSet a set of {key, value} pairs, where the key values are Long
|
||||
* objects. The elements of this set will be used to initialize
|
||||
* the key and value array column vectors that are children of
|
||||
* the MapColumnVector.
|
||||
* @param mapVector the MapColumnVector. This ColumnVector has children for the
|
||||
* key and value arrays.
|
||||
* @param fieldName the field name for the map column vector column.
|
||||
* @param rowNum the ORC file row number.
|
||||
*/
|
||||
private static void setLongKeyMap(Set<Map.Entry<Object, Object>> mapSet, MapColumnVector mapVector,
|
||||
String fieldName, int rowNum) {
|
||||
List<Object> keyValueList = mapSet.stream().map(Map.Entry::getKey).collect(Collectors.toList());
|
||||
if (isListType(keyValueList, isLong)) {
|
||||
LongColumnVector longVector = (LongColumnVector) mapVector.keys;
|
||||
int offset = (int) mapVector.offsets[rowNum];
|
||||
// set the key vector
|
||||
setLongListVector(keyValueList, longVector, offset, fieldName);
|
||||
// set the value vector
|
||||
ColumnVector valueVector = mapVector.values;
|
||||
List<Object> valueList = mapSet.stream().map(Map.Entry::getValue).collect(Collectors.toList());
|
||||
setMapValueVector(valueList, valueVector, offset, fieldName);
|
||||
} else {
|
||||
throw new RuntimeException("For field " + fieldName + " Long key type expected to match schema");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Initialize a MapColumnVector with Double key values.
|
||||
* </p>
|
||||
*
|
||||
* @param mapSet a set of {key, value} pairs, where the key values are Double
|
||||
* objects. The elements of this set will be used to initialize
|
||||
* the key and value array column vectors that are children of
|
||||
* the MapColumnVector.
|
||||
* @param mapVector the MapColumnVector. This ColumnVector has children for the
|
||||
* key and value arrays.
|
||||
* @param fieldName the field name for the map column vector column.
|
||||
* @param rowNum the ORC file row number.
|
||||
*/
|
||||
private static void setDoubleKeyMap(Set<Map.Entry<Object, Object>> mapSet, MapColumnVector mapVector,
|
||||
String fieldName, int rowNum) {
|
||||
List<Object> keyValueList = mapSet.stream().map(Map.Entry::getKey).collect(Collectors.toList());
|
||||
if (isListType(keyValueList, isDouble)) {
|
||||
DoubleColumnVector doubleVector = (DoubleColumnVector) mapVector.keys;
|
||||
int offset = (int) mapVector.offsets[rowNum];
|
||||
// set the key vector
|
||||
setDoubleListVector(keyValueList, doubleVector, offset, fieldName);
|
||||
// set the value vector
|
||||
ColumnVector valueVector = mapVector.values;
|
||||
List<Object> valueList = mapSet.stream().map(Map.Entry::getValue).collect(Collectors.toList());
|
||||
setMapValueVector(valueList, valueVector, offset, fieldName);
|
||||
} else {
|
||||
throw new RuntimeException("For field " + fieldName + " Long key type expected to match schema");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Initialize a MapColumnVector with String key values.
|
||||
* </p>
|
||||
*
|
||||
* @param mapSet a set of {key, value} pairs, where the key values are String
|
||||
* objects. The elements of this set will be used to initialize
|
||||
* the key and value array column vectors that are children of
|
||||
* the MapColumnVector.
|
||||
* @param mapVector the MapColumnVector. This ColumnVector has children for the
|
||||
* key and value arrays.
|
||||
* @param fieldName the field name for the map column vector column.
|
||||
* @param rowNum the ORC file row number.
|
||||
*/
|
||||
private static void setStringKeyMap(Set<Map.Entry<Object, Object>> mapSet, MapColumnVector mapVector,
|
||||
String fieldName, int rowNum) {
|
||||
List<Object> keyValueList = mapSet.stream().map(Map.Entry::getKey).collect(Collectors.toList());
|
||||
if (isListType(keyValueList, isString)) {
|
||||
BytesColumnVector byteVector = (BytesColumnVector) mapVector.keys;
|
||||
int offset = (int) mapVector.offsets[rowNum];
|
||||
// set the key array vector
|
||||
setBytesListVector(keyValueList, byteVector, offset, fieldName);
|
||||
// set the value array vector
|
||||
ColumnVector valueVector = mapVector.values;
|
||||
List<Object> valueList = mapSet.stream().map(Map.Entry::getValue).collect(Collectors.toList());
|
||||
setMapValueVector(valueList, valueVector, offset, fieldName);
|
||||
} else {
|
||||
throw new RuntimeException("For field " + fieldName + " Long key type expected to match schema");
|
||||
}
|
||||
}
|
||||
|
||||
private static void setMapVectorParameters(MapColumnVector mapVec, int maxBatchSize, int vecLength, int rowNum) {
|
||||
setMultiValuedVectorParameters(mapVec, vecLength, rowNum);
|
||||
mapVec.keys.ensureSize(maxBatchSize + vecLength, true);
|
||||
mapVec.values.ensureSize(maxBatchSize + vecLength, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Set the Map key and value elements for a MapColumnVector
|
||||
* </p>
|
||||
* <p>
|
||||
* A MapColumnVector has a single ColumnVector type for each of the map key and
|
||||
* map values. For example, the ColumnVector for the key values could be a
|
||||
* BytesColumnVector (a set of String keys). The values could be a
|
||||
* LongColumnVector.
|
||||
* </p>
|
||||
* <p>
|
||||
* In the documentation there is no restriction given for the map key type. This
|
||||
* code limits the key types to scalar values: string, long, double.
|
||||
* </p>
|
||||
* </p>
|
||||
* <p>
|
||||
* The documentation does not limit the map value types. This code limites the
|
||||
* map values to the same types that are supported for ListColumnVectors: long,
|
||||
* double, bytes (String), Decimal and Timestamp.
|
||||
* </p>
|
||||
*
|
||||
* @param colVal a HashMap object
|
||||
* @param typeDesc the schema description for the MapColumnVector column
|
||||
* @param fieldName the field name of the MapColumnVector column
|
||||
* @param mapVector The parent MapColumnVector
|
||||
* @param rowNum the ORC file column number.
|
||||
*/
|
||||
private static void setMapColumnVector(Object colVal, TypeDescription typeDesc, String fieldName,
|
||||
MapColumnVector mapVector, int rowNum) {
|
||||
if (colVal == null) {
|
||||
mapVector.isNull[rowNum] = true;
|
||||
mapVector.noNulls = false;
|
||||
} else {
|
||||
if (colVal instanceof HashMap) {
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<Object, Object> rawMap = (HashMap<Object, Object>) colVal;
|
||||
int mapLen = rawMap.size();
|
||||
final int maxBatchSize = typeDesc.createRowBatch().getMaxSize();
|
||||
setMapVectorParameters(mapVector, maxBatchSize, mapLen, rowNum);
|
||||
if (ORCCommonUtils.checkMapColumnVectorTypes(mapVector)) {
|
||||
Set<Map.Entry<Object, Object>> mapSet = rawMap.entrySet();
|
||||
switch (mapVector.keys.type) {
|
||||
case LONG:
|
||||
setLongKeyMap(mapSet, mapVector, fieldName, rowNum);
|
||||
break;
|
||||
case DOUBLE:
|
||||
setDoubleKeyMap(mapSet, mapVector, fieldName, rowNum);
|
||||
break;
|
||||
case BYTES:
|
||||
setStringKeyMap(mapSet, mapVector, fieldName, rowNum);
|
||||
break;
|
||||
default: {
|
||||
break;
|
||||
/* This block left intentionally empty */
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw new RuntimeException(
|
||||
"For field " + fieldName + " key types are limited to string, long and double. "
|
||||
+ "value types are limited to long, double, String, decimal and timestamp");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a column value in an ORC a row that will be written to the ORC file.
|
||||
*
|
||||
* @param colVal an Object containing the values to be written to the column
|
||||
* @param typeDesc the TypeDescription from the schema that defines the column
|
||||
* @param fieldName the column field name
|
||||
* @param vector the ColumnVector that will be initialized with the values in
|
||||
* the colVal argument.
|
||||
* @param rowNum the ORC file row number.
|
||||
*/
|
||||
public static void setColumn(Object colVal, TypeDescription typeDesc, String fieldName, ColumnVector vector,
|
||||
int rowNum) {
|
||||
if (colVal == null) {
|
||||
vector.isNull[rowNum] = true;
|
||||
vector.noNulls = false;
|
||||
} else {
|
||||
switch (vector.type) {
|
||||
case LONG: {
|
||||
if (vector instanceof DateColumnVector) {
|
||||
DateColumnVector dateVector = (DateColumnVector) vector;
|
||||
setDateColumnVector(colVal, fieldName, dateVector, rowNum);
|
||||
} else {
|
||||
LongColumnVector longVector = (LongColumnVector) vector;
|
||||
setLongColumnVector(colVal, fieldName, longVector, rowNum);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case DOUBLE: {
|
||||
DoubleColumnVector doubleVector = (DoubleColumnVector) vector;
|
||||
setDoubleVector(colVal, fieldName, doubleVector, rowNum);
|
||||
break;
|
||||
}
|
||||
case BYTES: {
|
||||
BytesColumnVector bytesColVector = (BytesColumnVector) vector;
|
||||
setByteColumnVector(colVal, fieldName, bytesColVector, rowNum);
|
||||
break;
|
||||
}
|
||||
case DECIMAL: {
|
||||
DecimalColumnVector decimalVector = (DecimalColumnVector) vector;
|
||||
setDecimalVector(colVal, fieldName, decimalVector, rowNum);
|
||||
break;
|
||||
}
|
||||
case DECIMAL_64:
|
||||
throw new RuntimeException("Field: " + fieldName + ", Decimal64ColumnVector is not supported");
|
||||
case TIMESTAMP: {
|
||||
TimestampColumnVector timestampVector = (TimestampColumnVector) vector;
|
||||
setTimestampVector(colVal, fieldName, timestampVector, rowNum);
|
||||
break;
|
||||
}
|
||||
case INTERVAL_DAY_TIME:
|
||||
throw new RuntimeException("Field: " + fieldName + ", HiveIntervalDayTime is not supported");
|
||||
case STRUCT: {
|
||||
StructColumnVector structVector = (StructColumnVector) vector;
|
||||
// setStructColumnVector(colVal, typeDesc, fieldName, structVector, rowNum);
|
||||
break;
|
||||
}
|
||||
case LIST: {
|
||||
ListColumnVector listVector = (ListColumnVector) vector;
|
||||
setListColumnVector(colVal, typeDesc, fieldName, listVector, rowNum);
|
||||
break;
|
||||
}
|
||||
case MAP: {
|
||||
MapColumnVector mapVector = (MapColumnVector) vector;
|
||||
setMapColumnVector(colVal, typeDesc, fieldName, mapVector, rowNum);
|
||||
break;
|
||||
}
|
||||
case UNION: {
|
||||
UnionColumnVector unionVector = (UnionColumnVector) vector;
|
||||
setUnionColumnVector(colVal, typeDesc, fieldName, unionVector, rowNum);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new RuntimeException("setColumn: Internal error: unexpected ColumnVector subtype");
|
||||
} // switch
|
||||
} // else
|
||||
} // setColumn
|
||||
|
||||
private static String orcExceptionMsg(String prefixMsg, String fieldName, int rowNum) {
|
||||
return prefixMsg + fieldName + " in row " + rowNum;
|
||||
}
|
||||
|
||||
public static TypeDescription detectType(Object value) {
|
||||
TypeDescription type = null;
|
||||
if (value != null) {
|
||||
if (value instanceof Boolean) {
|
||||
type = TypeDescription.createBoolean();
|
||||
} else if (value instanceof Short) {
|
||||
type = TypeDescription.createShort();
|
||||
} else if (value instanceof Integer) {
|
||||
type = TypeDescription.createInt();
|
||||
} else if (value instanceof Long) {
|
||||
type = TypeDescription.createLong();
|
||||
} else if (value instanceof Timestamp) {
|
||||
type = TypeDescription.createTimestamp();
|
||||
} else if (value instanceof BigDecimal) {
|
||||
type = TypeDescription.createDecimal();
|
||||
} else if (value instanceof Byte) {
|
||||
type = TypeDescription.createByte();
|
||||
} else if (value instanceof Float) {
|
||||
type = TypeDescription.createFloat();
|
||||
} else if (value instanceof Double) {
|
||||
type = TypeDescription.createDouble();
|
||||
} else if (value instanceof String) {
|
||||
type = TypeDescription.createString();
|
||||
} else if (value instanceof Date) {
|
||||
type = TypeDescription.createDate();
|
||||
} else if (value instanceof byte[]) {
|
||||
type = TypeDescription.createBinary();
|
||||
} else {
|
||||
throw new RuntimeException(
|
||||
value.getClass().getName() + " is not supported for ListColumnVector columns");
|
||||
}
|
||||
} else {
|
||||
type = TypeDescription.createString();
|
||||
}
|
||||
|
||||
return type;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
package org.talend.orc;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
|
||||
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
|
||||
import org.apache.orc.CompressionKind;
|
||||
import org.apache.orc.OrcFile;
|
||||
import org.apache.orc.OrcFile.WriterOptions;
|
||||
import org.apache.orc.Reader;
|
||||
import org.apache.orc.RecordReader;
|
||||
import org.apache.orc.TypeDescription;
|
||||
import org.apache.orc.Writer;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class ORCUtilsTest {
|
||||
|
||||
private static File localFolder;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() throws IOException {
|
||||
localFolder = createTempDirectory();
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
static void dispose() {
|
||||
localFolder.delete();
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that an exception is thrown if a Date type is written.
|
||||
*
|
||||
* At the time this test was written, the ORC writer did not correctly write the
|
||||
* date epoch value to the ORC file. The value was written as a 32-bit int,
|
||||
* instead of a 64 bit long. As a result, the date is incorrect. A timestamp
|
||||
* value should be used instead.
|
||||
*
|
||||
* @param tempDirPath
|
||||
* @throws ORCFileException
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
@Test
|
||||
void testAllDataTypes() throws Throwable {
|
||||
String filePath = localFolder.getAbsolutePath() + "/testAllTypes.orc";
|
||||
writeData(filePath);
|
||||
|
||||
readAndCheckData(filePath);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDetectType() {
|
||||
Assertions.assertEquals(TypeDescription.Category.BOOLEAN, ORCWriteUtils.detectType(true).getCategory());
|
||||
Assertions.assertEquals(TypeDescription.Category.SHORT,
|
||||
ORCWriteUtils.detectType(Short.valueOf("1")).getCategory());
|
||||
Assertions.assertEquals(TypeDescription.Category.INT, ORCWriteUtils.detectType(1).getCategory());
|
||||
Assertions.assertEquals(TypeDescription.Category.LONG, ORCWriteUtils.detectType(1L).getCategory());
|
||||
Assertions.assertEquals(TypeDescription.Category.TIMESTAMP,
|
||||
ORCWriteUtils.detectType(new Timestamp(System.currentTimeMillis())).getCategory());
|
||||
Assertions.assertEquals(TypeDescription.Category.DECIMAL,
|
||||
ORCWriteUtils.detectType(new BigDecimal("1")).getCategory());
|
||||
Assertions.assertEquals(TypeDescription.Category.BYTE,
|
||||
ORCWriteUtils.detectType(Byte.valueOf("1")).getCategory());
|
||||
Assertions.assertEquals(TypeDescription.Category.FLOAT, ORCWriteUtils.detectType(1.0f).getCategory());
|
||||
Assertions.assertEquals(TypeDescription.Category.DOUBLE, ORCWriteUtils.detectType(1.0).getCategory());
|
||||
Assertions.assertEquals(TypeDescription.Category.STRING, ORCWriteUtils.detectType("test").getCategory());
|
||||
Assertions.assertEquals(TypeDescription.Category.DATE, ORCWriteUtils.detectType(new Date()).getCategory());
|
||||
Assertions.assertEquals(TypeDescription.Category.BINARY,
|
||||
ORCWriteUtils.detectType("test".getBytes()).getCategory());
|
||||
}
|
||||
|
||||
private void writeData(String filePath) throws Throwable {
|
||||
TypeDescription schema = TypeDescription.createStruct();
|
||||
schema.addField("t_boolean", TypeDescription.createBoolean());
|
||||
schema.addField("t_byte", TypeDescription.createByte());
|
||||
schema.addField("t_bytes", TypeDescription.createBinary());
|
||||
schema.addField("t_char", TypeDescription.createChar());
|
||||
schema.addField("t_date", TypeDescription.createDate());
|
||||
schema.addField("t_ts", TypeDescription.createTimestamp());
|
||||
schema.addField("t_double", TypeDescription.createDouble());
|
||||
schema.addField("t_float", TypeDescription.createFloat());
|
||||
schema.addField("t_decimal", TypeDescription.createDecimal().withPrecision(18).withScale(5));
|
||||
schema.addField("t_int", TypeDescription.createInt());
|
||||
schema.addField("t_long", TypeDescription.createLong());
|
||||
schema.addField("t_short", TypeDescription.createShort());
|
||||
schema.addField("t_string", TypeDescription.createString());
|
||||
schema.addField("t_list", TypeDescription.createList(TypeDescription.createString()));
|
||||
|
||||
WriterOptions writerOption = OrcFile.writerOptions(new Configuration()) //
|
||||
.overwrite(true) //
|
||||
.compress(CompressionKind.valueOf("ZLIB")).setSchema(schema); //
|
||||
|
||||
Writer writer = OrcFile.createWriter(new Path(filePath), writerOption);
|
||||
VectorizedRowBatch batch = schema.createRowBatch(100);
|
||||
for (int r = 0; r < 1000; ++r) {
|
||||
int row = batch.size++;
|
||||
for (int i = 0; i < batch.cols.length; i++) {
|
||||
ColumnVector vector = batch.cols[i];
|
||||
TypeDescription type = schema.getChildren().get(i);
|
||||
switch (vector.type) {
|
||||
case BYTES:
|
||||
if (type.getCategory() == TypeDescription.Category.BINARY) {
|
||||
ORCWriteUtils.setColumn(("this is byte[] " + r).getBytes(), null, "t_bytes", vector, row);
|
||||
} else if (type.getCategory() == TypeDescription.Category.STRING) {
|
||||
if(r==666) {
|
||||
ORCWriteUtils.setColumn(null, null, "t_string", vector, row);
|
||||
}else {
|
||||
ORCWriteUtils.setColumn(("this is String " + r), null, "t_string", vector, row);
|
||||
}
|
||||
} else if (type.getCategory() == TypeDescription.Category.CHAR) {
|
||||
ORCWriteUtils.setColumn("talend".charAt(r % 6), null, "t_char", vector, row);
|
||||
} else {
|
||||
throw new RuntimeException(type.getCategory() + " is not supported as BYTES vector");
|
||||
}
|
||||
break;
|
||||
case DECIMAL:
|
||||
ORCWriteUtils.setColumn(new BigDecimal(r + ".12345"), null, "t_decimal", vector, row);
|
||||
break;
|
||||
case DOUBLE:
|
||||
if (type.getCategory() == TypeDescription.Category.DOUBLE) {
|
||||
ORCWriteUtils.setColumn(r + 0.123, null, "t_double", vector, row);
|
||||
} else if (type.getCategory() == TypeDescription.Category.FLOAT) {
|
||||
ORCWriteUtils.setColumn(r + 0.456f, null, "t_float", vector, row);
|
||||
} else {
|
||||
throw new RuntimeException(type.getCategory() + " is not supported as DOUBLE vector");
|
||||
}
|
||||
break;
|
||||
case LONG:
|
||||
if (type.getCategory() == TypeDescription.Category.BOOLEAN) {
|
||||
ORCWriteUtils.setColumn(true, null, "t_boolean", vector, row);
|
||||
} else if (type.getCategory() == TypeDescription.Category.BYTE) {
|
||||
ORCWriteUtils.setColumn((byte)(r % 128), null, "t_byte", vector, row);
|
||||
} else if (type.getCategory() == TypeDescription.Category.INT) {
|
||||
ORCWriteUtils.setColumn(r, null, "t_int", vector, row);
|
||||
} else if (type.getCategory() == TypeDescription.Category.SHORT) {
|
||||
ORCWriteUtils.setColumn((short)(r % 256), null, "t_short", vector, row);
|
||||
} else if (type.getCategory() == TypeDescription.Category.LONG) {
|
||||
ORCWriteUtils.setColumn(r * 1000L, null, "t_long", vector, row);
|
||||
} else if (type.getCategory() == TypeDescription.Category.DATE) {
|
||||
Date d = new Date(1633687854031L);
|
||||
ORCWriteUtils.setColumn(d, null, "t_date", vector, row);
|
||||
} else {
|
||||
throw new RuntimeException(type.getCategory() + " is not supported as LONG vector");
|
||||
}
|
||||
break;
|
||||
case TIMESTAMP:
|
||||
Timestamp ts = new java.sql.Timestamp(1633687854031L);
|
||||
ts.setNanos(123456789);
|
||||
ORCWriteUtils.setColumn(ts, null, "t_ts", vector, row);
|
||||
break;
|
||||
case LIST:
|
||||
List<String> values = new ArrayList<>();
|
||||
values.add("v1_" + r);
|
||||
values.add("v2_" + r);
|
||||
values.add("v3_" + r);
|
||||
ORCWriteUtils.setColumn(values, ORCWriteUtils.detectType("v1_" + r), "t_ list", vector, row);
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException(vector.type + " is not supported");
|
||||
|
||||
}
|
||||
}
|
||||
if (batch.size == batch.getMaxSize()) {
|
||||
writer.addRowBatch(batch);
|
||||
batch.reset();
|
||||
}
|
||||
}
|
||||
if (batch.size != 0) {
|
||||
writer.addRowBatch(batch);
|
||||
}
|
||||
|
||||
writer.close();
|
||||
}
|
||||
|
||||
private void readAndCheckData(String filePath) throws Throwable {
|
||||
|
||||
Reader reader = OrcFile.createReader(new Path(filePath), OrcFile.readerOptions(new Configuration()));
|
||||
TypeDescription schema = reader.getSchema();
|
||||
VectorizedRowBatch batch = schema.createRowBatch();
|
||||
RecordReader rowIterator = reader.rows(reader.options().schema(schema));
|
||||
int nuberLine = 0;
|
||||
List<Object> nb_500 = new ArrayList<>();
|
||||
List<Object> nb_666 = new ArrayList<>();
|
||||
while (rowIterator.nextBatch(batch)) {
|
||||
ColumnVector[] colVectors = batch.cols;
|
||||
for (int row = 0; row < batch.size; ++row) {
|
||||
nuberLine++;
|
||||
for (String columnName : schema.getFieldNames()) {
|
||||
ColumnVector colVector = colVectors[schema.getFieldNames().indexOf(columnName)];
|
||||
int colRow = colVector.isRepeating ? 0 : row;
|
||||
Object value = ORCReadUtils.readColumnByName(batch, columnName, schema, colRow);
|
||||
if (nuberLine == 500) {
|
||||
nb_500.add(value);
|
||||
}else if (nuberLine == 667) {
|
||||
nb_666.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Assertions.assertEquals(true, nb_500.get(0));
|
||||
Assertions.assertEquals(Byte.valueOf("115"), nb_500.get(1));
|
||||
Assertions.assertEquals("this is byte[] 499", new String((byte[]) nb_500.get(2)));
|
||||
Assertions.assertEquals("a", nb_500.get(3));
|
||||
Date t_date = (Date) nb_500.get(4);
|
||||
Assertions.assertEquals((1633687854000L/86400000), t_date.getTime()/86400000);
|
||||
Timestamp t_ts = (Timestamp) nb_500.get(5);
|
||||
Assertions.assertEquals(1633687854123L, t_ts.getTime());
|
||||
Assertions.assertEquals(123456789, t_ts.getNanos());
|
||||
Assertions.assertEquals(499.123, nb_500.get(6));
|
||||
Assertions.assertEquals(499.456f, (((float) nb_500.get(7)) * 1000) / 1000f);
|
||||
Assertions.assertEquals(new BigDecimal("499.12345"), nb_500.get(8));
|
||||
Assertions.assertEquals(499, nb_500.get(9));
|
||||
Assertions.assertEquals(499000L, nb_500.get(10));
|
||||
Assertions.assertEquals(Short.valueOf("243"), nb_500.get(11));
|
||||
Assertions.assertEquals("this is String 499", nb_500.get(12));
|
||||
Assertions.assertArrayEquals(Arrays.asList("v1_499", "v2_499", "v3_499").toArray(),
|
||||
((List<Object>) nb_500.get(13)).toArray());
|
||||
|
||||
//NB_LINE 666
|
||||
Assertions.assertNull( nb_666.get(12));
|
||||
|
||||
rowIterator.close();
|
||||
|
||||
}
|
||||
|
||||
public static File createTempDirectory() throws IOException {
|
||||
final File temp;
|
||||
|
||||
temp = File.createTempFile("temp", Long.toString(System.nanoTime()));
|
||||
if (!temp.delete()) {
|
||||
throw new IOException("Could not delete temp file: " + temp.getAbsolutePath());
|
||||
}
|
||||
|
||||
if (!temp.mkdir()) {
|
||||
throw new IOException("Could not create temp directory: " + temp.getAbsolutePath());
|
||||
}
|
||||
|
||||
return temp;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>talend-parquet</artifactId>
|
||||
<version>1.0</version>
|
||||
<version>1.3</version>
|
||||
|
||||
<properties>
|
||||
<parquet.version>1.10.1</parquet.version>
|
||||
|
||||
@@ -3,7 +3,9 @@ package org.talend.parquet.utils;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Calendar;
|
||||
import java.util.GregorianCalendar;
|
||||
import java.util.TimeZone;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.talend.parquet.data.simple.NanoTime;
|
||||
|
||||
@@ -11,6 +13,14 @@ import jodd.time.JulianDate;
|
||||
|
||||
public class NanoTimeUtils {
|
||||
|
||||
/**
|
||||
* Number of days between Julian day epoch (January 1, 4713 BC) and Unix day
|
||||
* epoch (January 1, 1970). The value of this constant is {@value}.
|
||||
*/
|
||||
public static final long JULIAN_EPOCH_OFFSET_DAYS = 2440588;
|
||||
private static final long MILLIS_IN_DAY = TimeUnit.DAYS.toMillis(1);
|
||||
private static final long NANOS_PER_MILLISECOND = TimeUnit.MILLISECONDS.toNanos(1);
|
||||
|
||||
static final long NANOS_PER_HOUR = java.util.concurrent.TimeUnit.HOURS.toNanos(1);
|
||||
static final long NANOS_PER_MINUTE = java.util.concurrent.TimeUnit.MINUTES.toNanos(1);
|
||||
static final long NANOS_PER_SECOND = java.util.concurrent.TimeUnit.SECONDS.toNanos(1);
|
||||
@@ -41,6 +51,31 @@ public class NanoTimeUtils {
|
||||
return calendar;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a timestamp to NanoTime.
|
||||
*/
|
||||
public static NanoTime getNanoTime(Timestamp ts, boolean skipConversion) {
|
||||
|
||||
Calendar calendar = getCalendar(skipConversion);
|
||||
calendar.setTimeInMillis(ts.getTime());
|
||||
int year = calendar.get(Calendar.YEAR);
|
||||
if (calendar.get(Calendar.ERA) == GregorianCalendar.BC) {
|
||||
year = 1 - year;
|
||||
}
|
||||
JulianDate jDateTime;
|
||||
jDateTime = JulianDate.of(year, calendar.get(Calendar.MONTH) + 1, // java calendar index starting at 1.
|
||||
calendar.get(Calendar.DAY_OF_MONTH), 0, 0, 0, 0);
|
||||
int days = jDateTime.getJulianDayNumber();
|
||||
|
||||
long hour = calendar.get(Calendar.HOUR_OF_DAY);
|
||||
long minute = calendar.get(Calendar.MINUTE);
|
||||
long second = calendar.get(Calendar.SECOND);
|
||||
long nanos = ts.getNanos();
|
||||
long nanosOfDay = nanos + NANOS_PER_SECOND * second + NANOS_PER_MINUTE * minute + NANOS_PER_HOUR * hour;
|
||||
|
||||
return new NanoTime(days, nanosOfDay);
|
||||
}
|
||||
|
||||
public static Timestamp getTimestamp(NanoTime nt, boolean skipConversion) {
|
||||
int julianDay = nt.getJulianDay();
|
||||
long nanosOfDay = nt.getTimeOfDayNanos();
|
||||
@@ -74,4 +109,23 @@ public class NanoTimeUtils {
|
||||
ts.setNanos((int) nanos);
|
||||
return ts;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns timestamp millis from NanoTime type value.
|
||||
*
|
||||
* @param nt NanoTime value
|
||||
* @return timestamp in millis
|
||||
*/
|
||||
public static long getTimestampMillis(NanoTime nt) {
|
||||
long timeOfDayNanos = nt.getTimeOfDayNanos();
|
||||
int julianDay = nt.getJulianDay();
|
||||
|
||||
return (julianDay - JULIAN_EPOCH_OFFSET_DAYS) * MILLIS_IN_DAY + (timeOfDayNanos / NANOS_PER_MILLISECOND);
|
||||
}
|
||||
|
||||
public static Timestamp getTimestamp(NanoTime nt) {
|
||||
Timestamp ts = new Timestamp(getTimestampMillis(nt));
|
||||
ts.setNanos((int) (nt.getTimeOfDayNanos() % 1000000000));
|
||||
return ts;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -195,31 +195,37 @@ public class TalendParquetUtils {
|
||||
}
|
||||
}
|
||||
|
||||
public static Binary decimalToBinary(BigDecimal decimalValue, int scale) {
|
||||
// First we need to make sure the BigDecimal matches our schema scale:
|
||||
decimalValue = decimalValue.setScale(scale, RoundingMode.HALF_UP);
|
||||
public static Binary decimalToBinary(BigDecimal decimalValue, int scale) {
|
||||
// First we need to make sure the BigDecimal matches our schema scale:
|
||||
decimalValue = decimalValue.setScale(scale, RoundingMode.HALF_UP);
|
||||
|
||||
// Next we get the decimal value as one BigInteger (like there was no decimal
|
||||
// point)
|
||||
BigInteger unscaledDecimalValue = decimalValue.unscaledValue();
|
||||
// Next we get the decimal value as one BigInteger (like there was no decimal
|
||||
// point)
|
||||
BigInteger unscaledDecimalValue = decimalValue.unscaledValue();
|
||||
|
||||
// Finally we serialize the integer
|
||||
byte[] decimalBytes = unscaledDecimalValue.toByteArray();
|
||||
byte[] decimalBuffer = null;
|
||||
// Finally we serialize the integer
|
||||
byte[] decimalBytes = unscaledDecimalValue.toByteArray();
|
||||
|
||||
byte[] decimalBuffer = new byte[16];
|
||||
if (decimalBuffer.length >= decimalBytes.length) {
|
||||
// Because we set our fixed byte array size as 16 bytes, we need to
|
||||
// pad-left our original value's bytes with zeros
|
||||
int decimalBufferIndex = decimalBuffer.length - 1;
|
||||
for (int i = decimalBytes.length - 1; i >= 0; i--) {
|
||||
decimalBuffer[decimalBufferIndex] = decimalBytes[i];
|
||||
decimalBufferIndex--;
|
||||
}
|
||||
} else {
|
||||
throw new IllegalArgumentException(String.format("Decimal size: %d was greater than the allowed max: %d",
|
||||
decimalBytes.length, decimalBuffer.length));
|
||||
}
|
||||
return Binary.fromReusedByteArray(decimalBuffer);
|
||||
}
|
||||
if (decimalValue.compareTo(BigDecimal.ZERO) < 0) {
|
||||
decimalBuffer = new byte[] { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 };
|
||||
} else {
|
||||
decimalBuffer = new byte[16];
|
||||
}
|
||||
if (decimalBuffer.length >= decimalBytes.length) {
|
||||
// Because we set our fixed byte array size as 16 bytes, we need to
|
||||
// pad-left our original value's bytes with 0 or -1
|
||||
int decimalBufferIndex = decimalBuffer.length - 1;
|
||||
for (int i = decimalBytes.length - 1; i >= 0; i--) {
|
||||
decimalBuffer[decimalBufferIndex] = decimalBytes[i];
|
||||
decimalBufferIndex--;
|
||||
}
|
||||
} else {
|
||||
throw new IllegalArgumentException(String
|
||||
.format("Decimal size: %d was greater than the allowed max: %d",
|
||||
decimalBytes.length, decimalBuffer.length));
|
||||
}
|
||||
return Binary.fromReusedByteArray(decimalBuffer);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -266,6 +266,21 @@ public class TalendParquetUtilsTest {
|
||||
group.append("aDecimal", TalendParquetUtils.decimalToBinary(decimalValue, 0));
|
||||
Assert.assertEquals(decimalValue, TalendParquetUtils.binaryToDecimal(group.getBinary(0, 0), 10, 0));
|
||||
|
||||
decimalValue = new BigDecimal("-93.5788130000");
|
||||
group = new SimpleGroup(schema);
|
||||
group.append("aDecimal", TalendParquetUtils.decimalToBinary(decimalValue, 10));
|
||||
Assert.assertEquals(decimalValue, TalendParquetUtils.binaryToDecimal(group.getBinary(0, 0), 38, 10));
|
||||
|
||||
decimalValue = new BigDecimal("-0.00");
|
||||
group = new SimpleGroup(schema);
|
||||
group.append("aDecimal", TalendParquetUtils.decimalToBinary(decimalValue, 2));
|
||||
Assert.assertEquals(decimalValue, TalendParquetUtils.binaryToDecimal(group.getBinary(0, 0), 38, 2));
|
||||
|
||||
decimalValue = new BigDecimal("0.000");
|
||||
group = new SimpleGroup(schema);
|
||||
group.append("aDecimal", TalendParquetUtils.decimalToBinary(decimalValue, 3));
|
||||
Assert.assertEquals(decimalValue, TalendParquetUtils.binaryToDecimal(group.getBinary(0, 0), 38, 3));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>talendzip</artifactId>
|
||||
<version>1.1-20201120</version>
|
||||
<version>1.2-20210901</version>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
@@ -54,17 +54,17 @@
|
||||
<artifactId>zip4j</artifactId>
|
||||
<version>1.3.3</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<version>1.19</version>
|
||||
<version>1.21</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>checkArchive-1.1-20190917</artifactId>
|
||||
<version>6.0.0</version>
|
||||
<groupId>org.talend.components</groupId>
|
||||
<artifactId>checkArchive</artifactId>
|
||||
<version>1.2-20210901</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
@@ -199,6 +199,23 @@
|
||||
<ITEM NAME="TRIM" FIELD="CHECK"></ITEM>
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="SET_QUERY_TIMEOUT"
|
||||
FIELD="CHECK"
|
||||
NUM_ROW="50"
|
||||
>
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="QUERY_TIMEOUT_IN_SECONDS"
|
||||
FIELD="TEXT"
|
||||
NUM_ROW="50"
|
||||
SHOW_IF="SET_QUERY_TIMEOUT == 'true'"
|
||||
>
|
||||
<DEFAULT>30</DEFAULT>
|
||||
</PARAMETER>
|
||||
</ADVANCED_PARAMETERS>
|
||||
|
||||
<CODEGENERATION>
|
||||
|
||||
@@ -22,6 +22,9 @@ DB_VERSION.ITEM.V6R1_V7R2=V6R1 to V7R2
|
||||
DB_VERSION.ITEM.V7R1_V7R3=V7R1 to V7R3
|
||||
MAPPING.NAME=Mapping
|
||||
|
||||
SET_QUERY_TIMEOUT.NAME=Set Query Timeout
|
||||
QUERY_TIMEOUT_IN_SECONDS.NAME=Timeout (seconds)
|
||||
|
||||
NB_LINE.NAME=Number of line
|
||||
GUESS_SCHEMA.NAME=Guess Schema
|
||||
CONNECTION.NAME=Component List
|
||||
|
||||
@@ -49,10 +49,14 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
String commitEvery = ElementParameterParser.getValue(node, "__COMMIT_EVERY__");
|
||||
|
||||
boolean isEnableDebug = ("true").equals(ElementParameterParser.getValue(node,"__ENABLE_DEBUG_MODE__"));
|
||||
|
||||
|
||||
boolean useBatchSize = "true".equals(ElementParameterParser.getValue(node,"__USE_BATCH_SIZE__"));
|
||||
String batchSize=ElementParameterParser.getValue(node,"__BATCH_SIZE__");
|
||||
|
||||
boolean setQueryTimeOut = "true".equals(ElementParameterParser.getValue(node,"__SET_QUERY_TIMEOUT__"));
|
||||
|
||||
String queryTimeOutInSeconds = ElementParameterParser.getValue(node,"__QUERY_TIMEOUT_IN_SECONDS__");
|
||||
|
||||
String rejectConnName = null;
|
||||
List<? extends IConnection> rejectConns = node.getOutgoingConnections("REJECT");
|
||||
if(rejectConns != null && rejectConns.size() > 0) {
|
||||
@@ -174,43 +178,47 @@ if(columnList != null && columnList.size() > 0) {
|
||||
StringBuilder updateSetStmt = actionSQLMap.get(UPDATE_SET_STMT);
|
||||
StringBuilder updateWhereStmt = actionSQLMap.get(UPDATE_WHERE_STMT);
|
||||
StringBuilder deleteWhereStmt = actionSQLMap.get(DELETE_WHERE_STMT);
|
||||
|
||||
%>
|
||||
java.sql.PreparedStatement pstmt_<%=cid %> = null;
|
||||
java.sql.PreparedStatement pstmtInsert_<%=cid %> = null;
|
||||
java.sql.PreparedStatement pstmtUpdate_<%=cid %> = null;
|
||||
<%
|
||||
if(("INSERT").equals(dataAction)) {
|
||||
%>
|
||||
String insert_<%=cid%> = "INSERT INTO " + <%=table%> + " (<%=insertColName.toString()%>) VALUES (<%=insertValueStmt.toString()%>)";
|
||||
java.sql.PreparedStatement pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(insert_<%=cid%>, java.sql.Statement.RETURN_GENERATED_KEYS);
|
||||
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(insert_<%=cid%>, java.sql.Statement.RETURN_GENERATED_KEYS);
|
||||
resourceMap.put("pstmt_<%=cid %>", pstmt_<%=cid %>);
|
||||
<%
|
||||
} else if (("UPDATE").equals(dataAction)) {
|
||||
%>
|
||||
String update_<%=cid%> = "UPDATE " + <%=table%> + " SET <%=updateSetStmt.toString()%> WHERE <%=updateWhereStmt.toString()%>";
|
||||
java.sql.PreparedStatement pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(update_<%=cid%>);
|
||||
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(update_<%=cid%>);
|
||||
resourceMap.put("pstmt_<%=cid %>", pstmt_<%=cid %>);
|
||||
<%
|
||||
} else if (("INSERT_OR_UPDATE").equals(dataAction)) {
|
||||
%>
|
||||
java.sql.PreparedStatement pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement("SELECT COUNT(1) FROM " + <%=table%> + " WHERE <%=updateWhereStmt.toString()%>");
|
||||
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement("SELECT COUNT(1) FROM " + <%=table%> + " WHERE <%=updateWhereStmt.toString()%>");
|
||||
resourceMap.put("pstmt_<%=cid %>", pstmt_<%=cid %>);
|
||||
String insert_<%=cid%> = "INSERT INTO " + <%=table%> + " (<%=insertColName.toString()%>) VALUES (<%=insertValueStmt.toString()%>)";
|
||||
java.sql.PreparedStatement pstmtInsert_<%=cid %> = conn_<%=cid%>.prepareStatement(insert_<%=cid%>);
|
||||
pstmtInsert_<%=cid %> = conn_<%=cid%>.prepareStatement(insert_<%=cid%>);
|
||||
resourceMap.put("pstmtInsert_<%=cid %>", pstmtInsert_<%=cid %>);
|
||||
String update_<%=cid%> = "UPDATE " + <%=table%> + " SET <%=updateSetStmt.toString()%> WHERE <%=updateWhereStmt.toString()%>";
|
||||
java.sql.PreparedStatement pstmtUpdate_<%=cid %> = conn_<%=cid%>.prepareStatement(update_<%=cid%>);
|
||||
pstmtUpdate_<%=cid %> = conn_<%=cid%>.prepareStatement(update_<%=cid%>);
|
||||
resourceMap.put("pstmtUpdate_<%=cid %>", pstmtUpdate_<%=cid %>);
|
||||
<%
|
||||
} else if (("UPDATE_OR_INSERT").equals(dataAction)) {
|
||||
%>
|
||||
String update_<%=cid%> = "UPDATE " + <%=table%> + " SET <%=updateSetStmt.toString()%> WHERE <%=updateWhereStmt.toString()%>";
|
||||
java.sql.PreparedStatement pstmtUpdate_<%=cid %> = conn_<%=cid%>.prepareStatement(update_<%=cid%>);
|
||||
pstmtUpdate_<%=cid %> = conn_<%=cid%>.prepareStatement(update_<%=cid%>);
|
||||
resourceMap.put("pstmtUpdate_<%=cid %>", pstmtUpdate_<%=cid %>);
|
||||
String insert_<%=cid%> = "INSERT INTO " + <%=table%> + " (<%=insertColName.toString()%>) VALUES (<%=insertValueStmt.toString()%>)";
|
||||
java.sql.PreparedStatement pstmtInsert_<%=cid %> = conn_<%=cid%>.prepareStatement(insert_<%=cid%>);
|
||||
pstmtInsert_<%=cid %> = conn_<%=cid%>.prepareStatement(insert_<%=cid%>);
|
||||
resourceMap.put("pstmtInsert_<%=cid %>", pstmtInsert_<%=cid %>);
|
||||
<%
|
||||
} else if (("DELETE").equals(dataAction)) {
|
||||
%>
|
||||
String delete_<%=cid%> = "DELETE FROM " + <%=table%> + " WHERE <%=deleteWhereStmt.toString()%>";
|
||||
java.sql.PreparedStatement pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(delete_<%=cid%>);
|
||||
pstmt_<%=cid %> = conn_<%=cid%>.prepareStatement(delete_<%=cid%>);
|
||||
resourceMap.put("pstmt_<%=cid %>", pstmt_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
@@ -220,6 +228,20 @@ if(columnList != null && columnList.size() > 0) {
|
||||
<%@ include file="../templates/DB/Output/splitSQLForAllDBInBegin.javajet" %>
|
||||
<%
|
||||
}
|
||||
if(setQueryTimeOut) {
|
||||
if(("INSERT").equals(dataAction) || ("UPDATE").equals(dataAction)
|
||||
|| ("DELETE").equals(dataAction) || ("INSERT_OR_UPDATE").equals(dataAction)) {
|
||||
%>
|
||||
pstmt_<%=cid %>.setQueryTimeout(<%= queryTimeOutInSeconds %>);
|
||||
<%
|
||||
}
|
||||
if(("UPDATE_OR_INSERT").equals(dataAction) || ("INSERT_OR_UPDATE").equals(dataAction)) {
|
||||
%>
|
||||
pstmtUpdate_<%=cid %>.setQueryTimeout(<%= queryTimeOutInSeconds %>);
|
||||
pstmtInsert_<%=cid %>.setQueryTimeout(<%= queryTimeOutInSeconds %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
} // end isDynamic
|
||||
if(isDynamic) {
|
||||
|
||||
@@ -322,6 +322,23 @@
|
||||
NUM_ROW="20">
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="SET_QUERY_TIMEOUT"
|
||||
FIELD="CHECK"
|
||||
NUM_ROW="30"
|
||||
>
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="QUERY_TIMEOUT_IN_SECONDS"
|
||||
FIELD="TEXT"
|
||||
NUM_ROW="30"
|
||||
SHOW_IF="SET_QUERY_TIMEOUT == 'true'"
|
||||
>
|
||||
<DEFAULT>30</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
</ADVANCED_PARAMETERS>
|
||||
|
||||
|
||||
@@ -41,10 +41,13 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
String batchSize =ElementParameterParser.getValue(node, "__BATCH_SIZE__");
|
||||
boolean useExistingConnection = "true".equals(ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__"));
|
||||
boolean isEnableDebug = ("true").equals(ElementParameterParser.getValue(node,"__ENABLE_DEBUG_MODE__"));
|
||||
|
||||
//feature:2880
|
||||
getManager(dbmsId, cid, node);
|
||||
boolean whereSupportNull = ElementParameterParser.getValue(node, "__SUPPORT_NULL_WHERE__").equals("true");
|
||||
|
||||
boolean setQueryTimeOut = "true".equals(ElementParameterParser.getValue(node,"__SET_QUERY_TIMEOUT__"));
|
||||
|
||||
String queryTimeOutInSeconds = ElementParameterParser.getValue(node,"__QUERY_TIMEOUT_IN_SECONDS__");
|
||||
|
||||
String incomingConnName = null;
|
||||
List<IMetadataColumn> columnList = getColumnList(node);
|
||||
@@ -227,6 +230,20 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
<%@ include file="../templates/DB/Output/splitSQLForAllDBInDynamicMain.javajet" %>
|
||||
<%
|
||||
}
|
||||
if(setQueryTimeOut) {
|
||||
if(("INSERT").equals(dataAction) || ("UPDATE").equals(dataAction)
|
||||
|| ("DELETE").equals(dataAction) || ("INSERT_OR_UPDATE").equals(dataAction)) {
|
||||
%>
|
||||
pstmt_<%=cid %>.setQueryTimeout(<%= queryTimeOutInSeconds %>);
|
||||
<%
|
||||
}
|
||||
if(("UPDATE_OR_INSERT").equals(dataAction) || ("INSERT_OR_UPDATE").equals(dataAction)) {
|
||||
%>
|
||||
pstmtUpdate_<%=cid %>.setQueryTimeout(<%= queryTimeOutInSeconds %>);
|
||||
pstmtInsert_<%=cid %>.setQueryTimeout(<%= queryTimeOutInSeconds %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
}
|
||||
|
||||
@@ -75,4 +75,7 @@ SUPPORT_NULL_WHERE.NAME=Support null in "SQL WHERE" statement
|
||||
CONNECTION.NAME=Component List
|
||||
USE_BATCH_SIZE.NAME=Use Batch Size
|
||||
BATCH_SIZE.NAME=Batch Size
|
||||
NB_LINE_REJECTED.NAME=NB Line Rejected
|
||||
NB_LINE_REJECTED.NAME=NB Line Rejected
|
||||
|
||||
SET_QUERY_TIMEOUT.NAME=Set Query Timeout
|
||||
QUERY_TIMEOUT_IN_SECONDS.NAME=Timeout (seconds)
|
||||
@@ -251,6 +251,23 @@
|
||||
REQUIRED="true" SHOW_IF="USE_EXISTING_CONNECTION == 'false'">
|
||||
<DEFAULT>10000</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="SET_QUERY_TIMEOUT"
|
||||
FIELD="CHECK"
|
||||
NUM_ROW="10"
|
||||
>
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="QUERY_TIMEOUT_IN_SECONDS"
|
||||
FIELD="TEXT"
|
||||
NUM_ROW="10"
|
||||
SHOW_IF="SET_QUERY_TIMEOUT == 'true'"
|
||||
>
|
||||
<DEFAULT>30</DEFAULT>
|
||||
</PARAMETER>
|
||||
</ADVANCED_PARAMETERS>
|
||||
|
||||
<CODEGENERATION>
|
||||
|
||||
@@ -52,4 +52,7 @@ SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Short=Short
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.String=String
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Time=Time
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Null=Null
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_VALUE=Parameter Value
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_VALUE=Parameter Value
|
||||
|
||||
SET_QUERY_TIMEOUT.NAME=Set Query Timeout
|
||||
QUERY_TIMEOUT_IN_SECONDS.NAME=Timeout (seconds)
|
||||
@@ -118,7 +118,7 @@
|
||||
<IMPORT NAME="Driver-JACKCESS" MODULE="jackcess-2.1.12.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess/2.1.12" REQUIRED="true"/>
|
||||
|
||||
<IMPORT NAME="jackcess-encrypt-2.1.4" MODULE="jackcess-encrypt-2.1.4.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess-encrypt/2.1.4" REQUIRED="true"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.68" MODULE="bcprov-jdk15on-1.68.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.68" REQUIRED="true"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.69" MODULE="bcprov-jdk15on-1.69.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.69" REQUIRED="true"/>
|
||||
<IMPORT NAME="talend-ucanaccess-utils-1.0.0" MODULE="talend-ucanaccess-utils-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-ucanaccess-utils-1.0.0/6.4.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.access/lib/talend-ucanaccess-utils-1.0.0.jar" REQUIRED="true"/>
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED="true"/>
|
||||
|
||||
@@ -121,7 +121,7 @@
|
||||
<IMPORT NAME="Driver-JACKCESS" MODULE="jackcess-2.1.12.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess/2.1.12" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="jackcess-encrypt-2.1.4" MODULE="jackcess-encrypt-2.1.4.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess-encrypt/2.1.4" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.68" MODULE="bcprov-jdk15on-1.68.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.68" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.69" MODULE="bcprov-jdk15on-1.69.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.69" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="talend-ucanaccess-utils-1.0.0" MODULE="talend-ucanaccess-utils-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-ucanaccess-utils-1.0.0/6.4.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.access/lib/talend-ucanaccess-utils-1.0.0.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
@@ -207,7 +207,7 @@
|
||||
<IMPORT NAME="Driver-JACKCESS" MODULE="jackcess-2.1.12.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess/2.1.12" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="jackcess-encrypt-2.1.4" MODULE="jackcess-encrypt-2.1.4.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess-encrypt/2.1.4" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.68" MODULE="bcprov-jdk15on-1.68.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.68" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.69" MODULE="bcprov-jdk15on-1.69.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.69" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="talend-ucanaccess-utils-1.0.0" MODULE="talend-ucanaccess-utils-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-ucanaccess-utils-1.0.0/6.4.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.access/lib/talend-ucanaccess-utils-1.0.0.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
@@ -181,7 +181,7 @@
|
||||
<IMPORT NAME="Driver-JACKCESS" MODULE="jackcess-2.1.12.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess/2.1.12" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="jackcess-encrypt-2.1.4" MODULE="jackcess-encrypt-2.1.4.jar" MVN="mvn:com.healthmarketscience.jackcess/jackcess-encrypt/2.1.4" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.68" MODULE="bcprov-jdk15on-1.68.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.68" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="bcprov-jdk15on-1.69" MODULE="bcprov-jdk15on-1.69.jar" MVN="mvn:org.bouncycastle/bcprov-jdk15on/1.69" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
<IMPORT NAME="talend-ucanaccess-utils-1.0.0" MODULE="talend-ucanaccess-utils-1.0.0.jar" MVN="mvn:org.talend.libraries/talend-ucanaccess-utils-1.0.0/6.4.0" UrlPath="platform:/plugin/org.talend.libraries.jdbc.access/lib/talend-ucanaccess-utils-1.0.0.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
<IMPORT NAME="common2.6" MODULE="commons-lang-2.6.jar" MVN="mvn:commons-lang/commons-lang/2.6" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false')"/>
|
||||
|
||||
@@ -388,9 +388,12 @@ class GenerateToolByDom4j{
|
||||
for(XMLNode attri:node.attributes){
|
||||
addAttribute(currEleName,attri);
|
||||
}
|
||||
if(node.name.indexOf(":")>0){
|
||||
int idx = node.name.indexOf(":");
|
||||
if(idx > 0){
|
||||
String prefix = node.name.substring(0, idx);
|
||||
String localName = node.name.substring(idx + 1);
|
||||
%>
|
||||
<%touchXMLNode.getXMLNode(currEleName);%>.setName("<%=node.name%>");
|
||||
<%touchXMLNode.getXMLElement(currEleName);%>.setQName(org.dom4j.DocumentHelper.createQName("<%=localName%>",<%touchXMLNode.getXMLElement(currEleName);%>.getNamespaceForPrefix("<%=prefix%>")));
|
||||
<%
|
||||
}
|
||||
int index = 0;
|
||||
|
||||
@@ -174,7 +174,7 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
if (nestRoot != null) {
|
||||
boolean isDefaultNameSpaceAtRoot = "".equals(nestRoot.getQName().getNamespace().getPrefix());
|
||||
for (org.dom4j.Element tmp: (java.util.List<org.dom4j.Element>) nestRoot.elements()) {
|
||||
if (("").equals(tmp.getQName().getNamespace().getURI()) && ("").equals(tmp.getQName().getNamespace().getPrefix()) && isDefaultNameSpaceAtRoot) {
|
||||
if (isDefaultNameSpaceAtRoot && ("").equals(tmp.getQName().getNamespace().getURI()) && ("").equals(tmp.getQName().getNamespace().getPrefix())) {
|
||||
tmp.setQName(org.dom4j.DocumentHelper.createQName(tmp.getName(), nestRoot.getQName().getNamespace()));
|
||||
}
|
||||
replaceDefaultNameSpace(tmp);
|
||||
|
||||
@@ -214,7 +214,7 @@
|
||||
<TEMPLATE_PARAM SOURCE="self.AUTO_COMMIT" TARGET="in.AUTO_COMMIT"/>
|
||||
</TEMPLATES>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-MysqlConnector" MODULE="mysql-connector-java-5.1.30-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.30-bin/6.0.0" />
|
||||
<IMPORT NAME="Driver-MysqlConnector" MODULE="mysql-connector-java-5.1.49.jar" MVN="mvn:mysql/mysql-connector-java/5.1.49" />
|
||||
<IMPORT NAME="slf4j-api-1.7.25.jar" MODULE="slf4j-api-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-api/1.7.25" REQUIRED_IF="(USE_SHARED_CONNECTION == 'true' AND SPECIFY_DATASOURCE_ALIAS=='false')" />
|
||||
<IMPORT NAME="slf4j-log4j12-1.7.25.jar" MODULE="slf4j-log4j12-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-log4j12/1.7.25" REQUIRED_IF="(USE_SHARED_CONNECTION == 'true' AND SPECIFY_DATASOURCE_ALIAS=='false')" />
|
||||
</IMPORTS>
|
||||
|
||||
@@ -298,7 +298,7 @@
|
||||
<TEMPLATE_PARAM SOURCE="self.TRIM" TARGET="in.TRIM"/>
|
||||
</TEMPLATES>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-Mysql5" MODULE="mysql-connector-java-5.1.30-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.30-bin/6.0.0" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="Driver-Mysql5" MODULE="mysql-connector-java-5.1.49.jar" MVN="mvn:mysql/mysql-connector-java/5.1.49" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -524,7 +524,7 @@
|
||||
<TEMPLATE_PARAM SOURCE="self.PARALLELIZE_NUMBER" TARGET="out.PARALLELIZE_NUMBER"/>
|
||||
</TEMPLATES>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-MysqlConnector" MODULE="mysql-connector-java-5.1.30-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.30-bin/6.0.0" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="Driver-MysqlConnector" MODULE="mysql-connector-java-5.1.49.jar" MVN="mvn:mysql/mysql-connector-java/5.1.49" REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.8 KiB |
@@ -0,0 +1,325 @@
|
||||
<COMPONENT>
|
||||
<HEADER
|
||||
PLATEFORM="ALL"
|
||||
SERIAL=""
|
||||
VERSION="0.102"
|
||||
STATUS="ALPHA"
|
||||
COMPATIBILITY="ALL"
|
||||
AUTHOR="Talend"
|
||||
RELEASE_DATE="20070314A"
|
||||
STARTABLE="true"
|
||||
DATA_AUTO_PROPAGATE="false"
|
||||
HAS_CONDITIONAL_OUTPUTS="true"
|
||||
SCHEMA_AUTO_PROPAGATE="false"
|
||||
PARTITIONING="AUTO"
|
||||
LOG4J_ENABLED="true"
|
||||
>
|
||||
<SIGNATURE/>
|
||||
</HEADER>
|
||||
|
||||
<FAMILIES>
|
||||
<FAMILY>Databases/DB Specifics/Amazon/Aurora</FAMILY>
|
||||
<FAMILY>Cloud/Amazon/Aurora</FAMILY>
|
||||
</FAMILIES>
|
||||
|
||||
<DOCUMENTATION>
|
||||
<URL/>
|
||||
</DOCUMENTATION>
|
||||
|
||||
<CONNECTORS>
|
||||
<CONNECTOR CTYPE="FLOW" MAX_INPUT="1" MAX_OUTPUT="1"/>
|
||||
<CONNECTOR NAME="REJECT" NOT_SHOW_IF="(DIE_ON_ERROR == 'true')" CTYPE="FLOW" MAX_INPUT="0" MAX_OUTPUT="1" LINE_STYLE="2" COLOR="FF0000" BASE_SCHEMA="FLOW" />
|
||||
<CONNECTOR CTYPE="ITERATE" MAX_OUTPUT="1" MAX_INPUT="1"/>
|
||||
<CONNECTOR CTYPE="SUBJOB_OK" MAX_INPUT="1" />
|
||||
<CONNECTOR CTYPE="SUBJOB_ERROR" MAX_INPUT="1" />
|
||||
<CONNECTOR CTYPE="COMPONENT_OK"/>
|
||||
<CONNECTOR CTYPE="COMPONENT_ERROR"/>
|
||||
<CONNECTOR CTYPE="RUN_IF"/>
|
||||
</CONNECTORS>
|
||||
|
||||
<PARAMETERS>
|
||||
<PARAMETER
|
||||
NAME="PROPERTY"
|
||||
FIELD="PROPERTY_TYPE"
|
||||
NUM_ROW="1"
|
||||
REPOSITORY_VALUE="DATABASE:AMAZON_AURORA"
|
||||
SHOW_IF="USE_EXISTING_CONNECTION == 'false'"
|
||||
/>
|
||||
|
||||
<PARAMETER
|
||||
NAME="DB_VERSION"
|
||||
FIELD="CLOSED_LIST"
|
||||
NUM_ROW="10"
|
||||
REPOSITORY_VALUE="DB_VERSION"
|
||||
READONLY="true"
|
||||
SHOW_IF="USE_EXISTING_CONNECTION == 'false'"
|
||||
SHOW="false">
|
||||
<ITEMS DEFAULT="MYSQL_5">
|
||||
<ITEM NAME="MYSQL_5" VALUE="MYSQL_5" />
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="USE_EXISTING_CONNECTION"
|
||||
FIELD="CHECK"
|
||||
NUM_ROW="20"
|
||||
>
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="CONNECTION"
|
||||
FIELD="COMPONENT_LIST"
|
||||
FILTER="tAmazonAuroraConnection"
|
||||
REQUIRED="true"
|
||||
NUM_ROW="20"
|
||||
SHOW_IF="USE_EXISTING_CONNECTION == 'true'"
|
||||
DYNAMIC_SETTINGS="true"
|
||||
/>
|
||||
|
||||
<PARAMETER
|
||||
NAME="HOST"
|
||||
FIELD="TEXT"
|
||||
NUM_ROW="30"
|
||||
REPOSITORY_VALUE="SERVER_NAME"
|
||||
REQUIRED="true"
|
||||
SHOW_IF="USE_EXISTING_CONNECTION == 'false'"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="PORT"
|
||||
FIELD="TEXT"
|
||||
NUM_ROW="30"
|
||||
REPOSITORY_VALUE="PORT"
|
||||
REQUIRED="true"
|
||||
SHOW_IF="USE_EXISTING_CONNECTION == 'false'"
|
||||
>
|
||||
<DEFAULT>"3306"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="DBNAME"
|
||||
FIELD="TEXT"
|
||||
NUM_ROW="40"
|
||||
REPOSITORY_VALUE="SID"
|
||||
REQUIRED="true"
|
||||
SHOW_IF="USE_EXISTING_CONNECTION == 'false'"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="TYPE"
|
||||
FIELD="TEXT"
|
||||
NUM_ROW="40"
|
||||
SHOW="false"
|
||||
REPOSITORY_VALUE="TYPE"
|
||||
>
|
||||
<DEFAULT>AMAZON_AURORA</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="USER"
|
||||
FIELD="TEXT"
|
||||
NUM_ROW="50"
|
||||
REPOSITORY_VALUE="USERNAME"
|
||||
REQUIRED="true"
|
||||
SHOW_IF="USE_EXISTING_CONNECTION == 'false'"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="PASS"
|
||||
FIELD="PASSWORD"
|
||||
NUM_ROW="50"
|
||||
REPOSITORY_VALUE="PASSWORD"
|
||||
REQUIRED="true"
|
||||
SHOW_IF="USE_EXISTING_CONNECTION == 'false'"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="SCHEMA_FLOW"
|
||||
FIELD="SCHEMA_TYPE"
|
||||
NUM_ROW="60"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="SCHEMA_REJECT"
|
||||
FIELD="SCHEMA_TYPE"
|
||||
REQUIRED="true"
|
||||
NUM_ROW="60"
|
||||
CONTEXT="REJECT"
|
||||
SHOW="true">
|
||||
<TABLE READONLY="true">
|
||||
<COLUMN NAME="errorCode" TYPE="id_String" LENGTH="255" READONLY="false" CUSTOM="true"/>
|
||||
<COLUMN NAME="errorMessage" TYPE="id_String" LENGTH="255" READONLY="false" CUSTOM="true"/>
|
||||
</TABLE>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="TABLE"
|
||||
FIELD="DBTABLE"
|
||||
NUM_ROW="60"
|
||||
LOG4J_ENABLED="false"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="QUERYSTORE"
|
||||
FIELD="QUERYSTORE_TYPE"
|
||||
NUM_ROW="70"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="QUERY"
|
||||
FIELD="MEMO_SQL"
|
||||
NUM_ROW="80"
|
||||
>
|
||||
<DEFAULT>"select id, name from employee"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="DIE_ON_ERROR"
|
||||
FIELD="CHECK"
|
||||
NUM_ROW="90"
|
||||
>
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
</PARAMETERS>
|
||||
|
||||
<ADVANCED_PARAMETERS>
|
||||
<PARAMETER NAME="PROPERTIES" FIELD="TEXT" NUM_ROW="4" REPOSITORY_VALUE="PROPERTIES_STRING" SHOW_IF="USE_EXISTING_CONNECTION == 'false'">
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="PROPAGATE_RECORD_SET"
|
||||
FIELD="CHECK"
|
||||
NUM_ROW="5"
|
||||
>
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="RECORD_SET_COLUMN"
|
||||
FIELD="COLUMN_LIST"
|
||||
NUM_ROW="5"
|
||||
SHOW_IF="PROPAGATE_RECORD_SET == 'true'"
|
||||
/>
|
||||
|
||||
<PARAMETER
|
||||
NAME="USE_PREPAREDSTATEMENT"
|
||||
FIELD="CHECK"
|
||||
NUM_ROW="6"
|
||||
>
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="SET_PREPAREDSTATEMENT_PARAMETERS"
|
||||
FIELD="TABLE"
|
||||
REQUIRED="false"
|
||||
NUM_ROW="7"
|
||||
NB_LINES="3"
|
||||
SHOW_IF="(USE_PREPAREDSTATEMENT == 'true')"
|
||||
>
|
||||
<ITEMS>
|
||||
<ITEM NAME="PARAMETER_INDEX" VALUE="" />
|
||||
<ITEM NAME="PARAMETER_TYPE" VALUE="String" FIELD="CLOSED_LIST">
|
||||
<ITEMS DEFAULT="String">
|
||||
<ITEM NAME="BigDecimal" VALUE="BigDecimal"/>
|
||||
<ITEM NAME="Blob" VALUE="Blob"/>
|
||||
<ITEM NAME="Boolean" VALUE="Boolean"/>
|
||||
<ITEM NAME="Byte" VALUE="Byte"/>
|
||||
<ITEM NAME="Bytes" VALUE="Bytes"/>
|
||||
<ITEM NAME="Clob" VALUE="Clob"/>
|
||||
<ITEM NAME="Date" VALUE="Date"/>
|
||||
<ITEM NAME="Double" VALUE="Double"/>
|
||||
<ITEM NAME="Float" VALUE="Float"/>
|
||||
<ITEM NAME="Int" VALUE="Int"/>
|
||||
<ITEM NAME="Long" VALUE="Long"/>
|
||||
<ITEM NAME="Object" VALUE="Object"/>
|
||||
<ITEM NAME="Short" VALUE="Short"/>
|
||||
<ITEM NAME="String" VALUE="String"/>
|
||||
<ITEM NAME="Time" VALUE="Time"/>
|
||||
<ITEM NAME="Null" VALUE="Null"/>
|
||||
</ITEMS>
|
||||
</ITEM>
|
||||
<ITEM NAME="PARAMETER_VALUE" VALUE="" />
|
||||
</ITEMS>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="ENCODING"
|
||||
FIELD="ENCODING_TYPE"
|
||||
NUM_ROW="9"
|
||||
REQUIRED="true"
|
||||
SHOW="false"
|
||||
>
|
||||
<DEFAULT>"ISO-8859-15"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
<PARAMETER
|
||||
NAME="COMMIT_EVERY"
|
||||
FIELD="TEXT"
|
||||
NUM_ROW="9"
|
||||
REQUIRED="true"
|
||||
SHOW_IF="USE_EXISTING_CONNECTION == 'false'"
|
||||
>
|
||||
<DEFAULT>10000</DEFAULT>
|
||||
</PARAMETER>
|
||||
</ADVANCED_PARAMETERS>
|
||||
|
||||
<CODEGENERATION>
|
||||
<TEMPLATES INPUT="in" OUTPUT="out">
|
||||
<TEMPLATE NAME="in" COMPONENT="tMysqlRow">
|
||||
<LINK_TO NAME="out" CTYPE="FLOW"/>
|
||||
</TEMPLATE>
|
||||
<TEMPLATE NAME="out" COMPONENT="tDummyRow"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.PROPERTY" TARGET="in.PROPERTY"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.DB_VERSION" TARGET="in.DB_VERSION"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.USE_EXISTING_CONNECTION" TARGET="in.USE_EXISTING_CONNECTION"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.CONNECTION" TARGET="in.CONNECTION"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.HOST" TARGET="in.HOST"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.PORT" TARGET="in.PORT"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.DBNAME" TARGET="in.DBNAME"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.TYPE" TARGET="in.TYPE"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.USER" TARGET="in.USER"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.PASS" TARGET="in.PASS"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.SCHEMA_FLOW" TARGET="in.SCHEMA_FLOW"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.SCHEMA_REJECT" TARGET="in.SCHEMA_REJECT"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.TABLE" TARGET="in.TABLE"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.QUERYSTORE" TARGET="in.QUERYSTORE"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.QUERY" TARGET="in.QUERY"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.DIE_ON_ERROR" TARGET="in.DIE_ON_ERROR"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.PROPERTIES" TARGET="in.PROPERTIES"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.PROPAGATE_RECORD_SET" TARGET="in.PROPAGATE_RECORD_SET"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.RECORD_SET_COLUMN" TARGET="in.RECORD_SET_COLUMN"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.USE_PREPAREDSTATEMENT" TARGET="in.USE_PREPAREDSTATEMENT"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.SET_PREPAREDSTATEMENT_PARAMETERS" TARGET="in.SET_PREPAREDSTATEMENT_PARAMETERS"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.PARAMETER_INDEX" TARGET="in.PARAMETER_INDEX"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.PARAMETER_TYPE" TARGET="in.PARAMETER_TYPE"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.PARAMETER_VALUE" TARGET="in.PARAMETER_VALUE"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.ENCODING" TARGET="in.ENCODING"/>
|
||||
<TEMPLATE_PARAM SOURCE="self.COMMIT_EVERY" TARGET="in.COMMIT_EVERY"/>
|
||||
|
||||
</TEMPLATES>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-Mysql5" MODULE="mysql-connector-java-5.1.49.jar" MVN="mvn:mysql/mysql-connector-java/5.1.49" REQUIRED_IF="(DB_VERSION == 'MYSQL_5') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
<RETURNS>
|
||||
<RETURN NAME="QUERY" TYPE="id_String" AVAILABILITY="FLOW"/>
|
||||
</RETURNS>
|
||||
</COMPONENT>
|
||||
@@ -0,0 +1,60 @@
|
||||
#Created by JInto - www.guh-software.de
|
||||
#Thu Aug 30 19:54:54 CST 2007
|
||||
COMMIT_EVERY.NAME=Commit every
|
||||
DBD-mysql.INFO=Required for MySQL
|
||||
DBNAME.NAME=Database
|
||||
DBTABLE.NAME=Table Name
|
||||
DIE_ON_ERROR.NAME=Die on error
|
||||
ENCODING.NAME=Encoding
|
||||
HELP=org.talend.help.tAmazonAuroraRow
|
||||
HOST.NAME=Host
|
||||
LONG_NAME=Executes an SQL query at each of Talend flow's iterations
|
||||
NB_LINE.NAME=Number of line
|
||||
PASS.NAME=Password
|
||||
PORT.NAME=Port
|
||||
PREPARED_STATEMENT.NAME=Prepared statement
|
||||
QUERY.NAME=Query
|
||||
REJECT.LINK=Rejects
|
||||
REJECT.MENU=Rejects
|
||||
SCHEMA_DB.NAME=Schema
|
||||
STATEMENT_PARAMS.ITEM.INPUT_COLUMN=Input column
|
||||
STATEMENT_PARAMS.NAME=Parameters
|
||||
TYPE.ITEM.MSSQL_ODBC=Microsoft SQL (Odbc driver)
|
||||
TYPE.ITEM.MYSQL=MySQL
|
||||
TYPE.NAME=Database Driver
|
||||
USER.NAME=Username
|
||||
USE_EXISTING_CONNECTION.NAME=Use an existing connection
|
||||
PROPERTIES.NAME=Additional JDBC Parameters
|
||||
PROPAGATE_RECORD_SET.NAME=Propagate QUERY's recordset
|
||||
RECORD_SET_COLUMN.NAME=use column
|
||||
TABLE.NAME=Table Name
|
||||
PROPERTY.NAME=Property Type
|
||||
CONNECTION.NAME=Component List
|
||||
SCHEMA_FLOW.NAME=Schema
|
||||
SCHEMA_REJECT.NAME=Reject Schema
|
||||
QUERYSTORE.NAME=Query Type
|
||||
|
||||
DB_VERSION.NAME=DB Version
|
||||
DB_VERSION.ITEM.MYSQL_5=Mysql 5
|
||||
|
||||
USE_PREPAREDSTATEMENT.NAME=Use PreparedStatement
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.NAME=Set PreparedStatement Parameters
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_INDEX=Parameter Index
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE=Parameter Type
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.BigDecimal=BigDecimal
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Blob=Blob
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Boolean=Boolean
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Byte=Byte
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Bytes=Bytes
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Clob=Clob
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Date=Date
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Double=Double
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Float=Float
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Int=Int
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Long=Long
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Object=Object
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Short=Short
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.String=String
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Time=Time
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_TYPE.ITEM.Null=Null
|
||||
SET_PREPAREDSTATEMENT_PARAMETERS.ITEM.PARAMETER_VALUE=Parameter Value
|
||||
@@ -11,25 +11,10 @@ imports="
|
||||
|
||||
public void createURL(INode node) {
|
||||
super.createURL(node);
|
||||
final boolean supportBulkComponent = false;
|
||||
final String jdbcURL = "jdbc:mysql";
|
||||
%>
|
||||
<%
|
||||
if(dbproperties == null || ("\"\"").equals(dbproperties) || ("").equals(dbproperties)) {
|
||||
%>
|
||||
String url_<%=cid%> = "jdbc:mysql://" + <%=dbhost%> + ":" + <%=dbport%> + "/" + <%=dbname%> + "?" + "rewriteBatchedStatements=true";
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
String proprtties_<%=cid%> = <%=dbproperties%>;
|
||||
if (proprtties_<%=cid%> == null || proprtties_<%=cid%>.trim().length() == 0) {
|
||||
proprtties_<%=cid%> += "rewriteBatchedStatements=true";
|
||||
}else if (proprtties_<%=cid%> != null && !proprtties_<%=cid%>.contains("rewriteBatchedStatements")) {
|
||||
proprtties_<%=cid%> += "&rewriteBatchedStatements=true";
|
||||
}
|
||||
|
||||
String url_<%=cid%> = "jdbc:mysql://" + <%=dbhost%> + ":" + <%=dbport%> + "/" + <%=dbname%> + "?" + proprtties_<%=cid%>;
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/Mysql/jdbcurl4connection_output.javajet"%>
|
||||
<%
|
||||
}
|
||||
|
||||
|
||||
@@ -162,7 +162,7 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-MysqlConnector" MODULE="mysql-connector-java-5.1.30-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.30-bin/6.0.0" REQUIRED_IF="DB_VERSION == 'MYSQL_5'" />
|
||||
<IMPORT NAME="Driver-MysqlConnector" MODULE="mysql-connector-java-5.1.49.jar" MVN="mvn:mysql/mysql-connector-java/5.1.49" REQUIRED_IF="DB_VERSION == 'MYSQL_5'" />
|
||||
<IMPORT NAME="Driver-MysqlConnector_Old" MODULE="mysql-connector-java-5.1.0-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.0-bin/6.0.0" REQUIRED="false" />
|
||||
<IMPORT NAME="slf4j-api-1.7.25.jar" MODULE="slf4j-api-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-api/1.7.25" REQUIRED_IF="USE_SHARED_CONNECTION =='true'" />
|
||||
<IMPORT NAME="slf4j-log4j12-1.7.25.jar" MODULE="slf4j-log4j12-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-log4j12/1.7.25" REQUIRED_IF="USE_SHARED_CONNECTION =='true'" />
|
||||
|
||||
@@ -28,16 +28,10 @@ imports="
|
||||
}
|
||||
|
||||
public void setURL(INode node) {
|
||||
|
||||
if(dbproperties == null || ("\"\"").equals(dbproperties) || ("").equals(dbproperties)) {
|
||||
final String jdbcURL = "jdbc:mysql";
|
||||
%>
|
||||
String url_<%=cid%> = "jdbc:mysql://" + <%=dbhost%> + ":" + <%=dbport%> + "/" + <%=dbname%>;
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/Mysql/jdbcurl4basic.javajet"%>
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
String url_<%=cid%> = "jdbc:mysql://" + <%=dbhost%> + ":" + <%=dbport%> + "/" + <%=dbname%> + "?" + <%=dbproperties%>;
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
public String getDirverClassName(INode node){
|
||||
|
||||
@@ -240,7 +240,7 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-Mysql5" MODULE="mysql-connector-java-5.1.30-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.30-bin/6.0.0" REQUIRED_IF="(DB_VERSION == 'MYSQL_5') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-Mysql5" MODULE="mysql-connector-java-5.1.49.jar" MVN="mvn:mysql/mysql-connector-java/5.1.49" REQUIRED_IF="(DB_VERSION == 'MYSQL_5') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -157,19 +157,13 @@ if(useExistingConnection) {
|
||||
<%
|
||||
} else {
|
||||
String javaDbDriver = "org.gjt.mm.mysql.Driver";
|
||||
|
||||
final boolean supportBulkComponent = false;
|
||||
final String jdbcURL = "jdbc:mysql";
|
||||
%>
|
||||
String dbProperties_<%=cid%> = <%=dbproperties%>;
|
||||
String url_<%=cid%> = null;
|
||||
if(dbProperties_<%=cid%> == null || dbProperties_<%=cid%>.trim().length() == 0) {
|
||||
url_<%=cid%> = "jdbc:mysql://" + <%=dbhost%> + ":" + <%=dbport%> + "/" + <%=dbname%> + "?" + "rewriteBatchedStatements=true";
|
||||
} else {
|
||||
String properties_<%=cid%> = <%=dbproperties%>;
|
||||
if (!properties_<%=cid%>.contains("rewriteBatchedStatements")) {
|
||||
properties_<%=cid%> += "&rewriteBatchedStatements=true";
|
||||
}
|
||||
|
||||
url_<%=cid%> = "jdbc:mysql://" + <%=dbhost%> + ":" + <%=dbport%> + "/" + <%=dbname%> + "?" + properties_<%=cid%>;
|
||||
}
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/Mysql/jdbcurl4connection_output.javajet"%>
|
||||
|
||||
String dbUser_<%=cid %> = <%=dbuser%>;
|
||||
|
||||
<%
|
||||
|
||||
@@ -423,7 +423,7 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-Mysql5" MODULE="mysql-connector-java-5.1.30-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.30-bin/6.0.0" REQUIRED_IF="(DB_VERSION == 'MYSQL_5') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-Mysql5" MODULE="mysql-connector-java-5.1.49.jar" MVN="mvn:mysql/mysql-connector-java/5.1.49" REQUIRED_IF="(DB_VERSION == 'MYSQL_5') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -16,16 +16,10 @@ imports="
|
||||
}
|
||||
|
||||
public void setURL(INode node) {
|
||||
|
||||
if(dbproperties == null || ("\"\"").equals(dbproperties) || ("").equals(dbproperties)) {
|
||||
final String jdbcURL = "jdbc:mysql";
|
||||
%>
|
||||
String url_<%=cid%> = "jdbc:mysql://" + <%=dbhost%> + ":" + <%=dbport%> + "/" + <%=dbname%>;
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/Mysql/jdbcurl4basic.javajet"%>
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
String url_<%=cid%> = "jdbc:mysql://" + <%=dbhost%> + ":" + <%=dbport%> + "/" + <%=dbname%> + "?" + <%=dbproperties%>;
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
public String getDirverClassName(INode node){
|
||||
|
||||
@@ -281,7 +281,7 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-Mysql5" MODULE="mysql-connector-java-5.1.30-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.30-bin/6.0.0" REQUIRED_IF="(DB_VERSION == 'MYSQL_5') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-Mysql5" MODULE="mysql-connector-java-5.1.49.jar" MVN="mvn:mysql/mysql-connector-java/5.1.49" REQUIRED_IF="(DB_VERSION == 'MYSQL_5') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -381,11 +381,12 @@
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-mssql-jdbc" MODULE="mssql-jdbc.jar" MVN="mvn:org.talend.libraries/mssql-jdbc/6.0.0" REQUIRED_IF="DRIVER=='MSSQL_PROP'" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.5.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.5" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.7.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="commons-lang3" MODULE="commons-lang3-3.10.jar" MVN="mvn:org.apache.commons/commons-lang3/3.10" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="commons-codec" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="gson" MODULE="gson-2.8.6.jar" MVN="mvn:com.google.code.gson/gson/2.8.6" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-6.5.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/6.5" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-9.7.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/9.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="content-type" MODULE="content-type-2.1.jar" MVN="mvn:com.nimbusds/content-type/2.1" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="json-smart" MODULE="json-smart-2.4.7.jar" MVN="mvn:net.minidev/json-smart/2.4.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="nimbus-jose-jwt" MODULE="nimbus-jose-jwt-8.11.jar" MVN="mvn:com.nimbusds/nimbus-jose-jwt/8.11" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="javax.mail" MODULE="javax.mail-1.6.2.jar" MVN="mvn:com.sun.mail/javax.mail/1.6.2" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
|
||||
@@ -204,11 +204,12 @@
|
||||
<IMPORT NAME="Driver-mssql-jdbc" MODULE="mssql-jdbc.jar" MVN="mvn:org.talend.libraries/mssql-jdbc/6.0.0" REQUIRED_IF="DRIVER=='MSSQL_PROP'" />
|
||||
<IMPORT NAME="slf4j-api-1.7.25.jar" MODULE="slf4j-api-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-api/1.7.25" REQUIRED_IF="(USE_SHARED_CONNECTION == 'true' AND SPECIFY_DATASOURCE_ALIAS=='false')" />
|
||||
<IMPORT NAME="slf4j-log4j12-1.7.25.jar" MODULE="slf4j-log4j12-1.7.25.jar" MVN="mvn:org.slf4j/slf4j-log4j12/1.7.25" REQUIRED_IF="(USE_SHARED_CONNECTION == 'true' AND SPECIFY_DATASOURCE_ALIAS=='false')" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.5.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.5" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.7.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="commons-lang3" MODULE="commons-lang3-3.10.jar" MVN="mvn:org.apache.commons/commons-lang3/3.10" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="commons-codec" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="gson" MODULE="gson-2.8.6.jar" MVN="mvn:com.google.code.gson/gson/2.8.6" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-6.5.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/6.5" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-9.7.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/9.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="content-type" MODULE="content-type-2.1.jar" MVN="mvn:com.nimbusds/content-type/2.1" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="json-smart" MODULE="json-smart-2.4.7.jar" MVN="mvn:net.minidev/json-smart/2.4.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="nimbus-jose-jwt" MODULE="nimbus-jose-jwt-8.11.jar" MVN="mvn:com.nimbusds/nimbus-jose-jwt/8.11" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="javax.mail" MODULE="javax.mail-1.6.2.jar" MVN="mvn:com.sun.mail/javax.mail/1.6.2" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true')" />
|
||||
|
||||
@@ -264,11 +264,12 @@
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-mssql-jdbc" MODULE="mssql-jdbc.jar" MVN="mvn:org.talend.libraries/mssql-jdbc/6.0.0" REQUIRED_IF="(DRIVER == 'MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Talend_MssqlUtil" MODULE="talend_DB_mssqlUtil-1.4.jar" MVN="mvn:org.talend.components.lib/talend_DB_mssqlUtil/1.4" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talend_DB_mssqlUtil-1.4.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.5.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.5" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.7.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="commons-lang3" MODULE="commons-lang3-3.10.jar" MVN="mvn:org.apache.commons/commons-lang3/3.10" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="commons-codec" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="gson" MODULE="gson-2.8.6.jar" MVN="mvn:com.google.code.gson/gson/2.8.6" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-6.5.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/6.5" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-9.7.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/9.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="content-type" MODULE="content-type-2.1.jar" MVN="mvn:com.nimbusds/content-type/2.1" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="json-smart" MODULE="json-smart-2.4.7.jar" MVN="mvn:net.minidev/json-smart/2.4.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="nimbus-jose-jwt" MODULE="nimbus-jose-jwt-8.11.jar" MVN="mvn:com.nimbusds/nimbus-jose-jwt/8.11" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="javax.mail" MODULE="javax.mail-1.6.2.jar" MVN="mvn:com.sun.mail/javax.mail/1.6.2" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
|
||||
@@ -335,11 +335,12 @@
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-mssql-jdbc" MODULE="mssql-jdbc.jar" MVN="mvn:org.talend.libraries/mssql-jdbc/6.0.0" REQUIRED_IF="(DRIVER == 'MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.5.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.5" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.7.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="commons-lang3" MODULE="commons-lang3-3.10.jar" MVN="mvn:org.apache.commons/commons-lang3/3.10" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="commons-codec" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="gson" MODULE="gson-2.8.6.jar" MVN="mvn:com.google.code.gson/gson/2.8.6" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-6.5.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/6.5" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-9.7.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/9.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="content-type" MODULE="content-type-2.1.jar" MVN="mvn:com.nimbusds/content-type/2.1" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="json-smart" MODULE="json-smart-2.4.7.jar" MVN="mvn:net.minidev/json-smart/2.4.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="nimbus-jose-jwt" MODULE="nimbus-jose-jwt-8.11.jar" MVN="mvn:com.nimbusds/nimbus-jose-jwt/8.11" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="javax.mail" MODULE="javax.mail-1.6.2.jar" MVN="mvn:com.sun.mail/javax.mail/1.6.2" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
|
||||
@@ -320,11 +320,12 @@
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-mssql-jdbc" MODULE="mssql-jdbc.jar" MVN="mvn:org.talend.libraries/mssql-jdbc/6.0.0" REQUIRED_IF="(DRIVER=='MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.5.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.5" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.7.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="commons-lang3" MODULE="commons-lang3-3.10.jar" MVN="mvn:org.apache.commons/commons-lang3/3.10" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="commons-codec" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="gson" MODULE="gson-2.8.6.jar" MVN="mvn:com.google.code.gson/gson/2.8.6" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-6.5.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/6.5" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-9.7.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/9.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="content-type" MODULE="content-type-2.1.jar" MVN="mvn:com.nimbusds/content-type/2.1" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="json-smart" MODULE="json-smart-2.4.7.jar" MVN="mvn:net.minidev/json-smart/2.4.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="nimbus-jose-jwt" MODULE="nimbus-jose-jwt-8.11.jar" MVN="mvn:com.nimbusds/nimbus-jose-jwt/8.11" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="javax.mail" MODULE="javax.mail-1.6.2.jar" MVN="mvn:com.sun.mail/javax.mail/1.6.2" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
|
||||
@@ -314,7 +314,7 @@
|
||||
<IMPORT NAME="google-api-services-storage-v1-rev20201112-1.31.0" MODULE="google-api-services-storage-v1-rev20201112-1.31.0.jar" MVN="mvn:com.google.apis/google-api-services-storage/v1-rev20201112-1.31.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="gson-2.8.6" MODULE="gson-2.8.6.jar" MVN="mvn:com.google.code.gson/gson/2.8.6" REQUIRED="true" />
|
||||
<IMPORT NAME="javax.annotation-api" MODULE="javax.annotation-api-1.3.jar" MVN="mvn:javax.annotation/javax.annotation-api/1.3" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="protobuf-java-2.5.0" MODULE="protobuf-java-2.5.0.jar" MVN="mvn:com.google.protobuf/protobuf-java/2.5.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="protobuf-java-3.4.0" MODULE="protobuf-java-3.4.0.jar" MVN="mvn:com.google.protobuf/protobuf-java/3.4.0" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="error_prone_annotation-2.1.3" MODULE="error_prone_annotation-2.1.3.jar" MVN="mvn:com.google.errorprone/error_prone_annotations/2.1.3" REQUIRED_IF="(AUTH_TYPE == 'GS_SERVICE_ACCOUNT') OR (AUTH_TYPE == 'TOKEN')" />
|
||||
<IMPORT NAME="opencensus-api-0.24.0" MODULE="opencensus-api-0.24.0.jar" MVN="mvn:io.opencensus/opencensus-api/0.24.0" REQUIRED="true" />
|
||||
<IMPORT NAME="opencensus-contrib-http-util-0.24.0" MODULE="opencensus-contrib-http-util-0.24.0.jar" MVN="mvn:io.opencensus/opencensus-contrib-http-util/0.24.0" REQUIRED="true" />
|
||||
|
||||
@@ -257,7 +257,9 @@ class BigQueryUtil_<%=cid%> {
|
||||
<%
|
||||
}
|
||||
%>
|
||||
jobList.add(job);
|
||||
if (job.getStatistics().getScriptStatistics() == null || !"EXPRESSION".equals(job.getStatistics().getScriptStatistics().getEvaluationKind())){
|
||||
jobList.add(job);
|
||||
}
|
||||
statistics_<%=cid %>.add(job.getStatistics().toString());
|
||||
}
|
||||
java.util.Collections.reverse(jobList);
|
||||
|
||||
@@ -431,8 +431,11 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",ee_<%=cid%>.getMessage());
|
||||
%>
|
||||
continue;
|
||||
}
|
||||
|
||||
if (job_iterable_<%=cid%>.getStatistics().getScriptStatistics() != null && "EXPRESSION".equals(job_iterable_<%=cid%>.getStatistics().getScriptStatistics().getEvaluationKind())){
|
||||
continue;
|
||||
}
|
||||
com.google.cloud.bigquery.TableResult result_<%=cid%> = job_iterable_<%=cid%>.getQueryResults();
|
||||
|
||||
//Dynamic start
|
||||
|
||||
<%
|
||||
|
||||
@@ -271,6 +271,9 @@
|
||||
<PARAMETER NAME="TOKEN_NAME" FIELD="FILE" NUM_ROW="10"
|
||||
REQUIRED="true" SHOW_IF="AUTH_MODE == 'OAUTH'">
|
||||
<DEFAULT>"__COMP_DEFAULT_FILE_DIR__/token.properties"</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="USE_12_HOUR_FORMAT" FIELD="CHECK" REQUIRED="true" NUM_ROW="15" REPOSITORY_VALUE="USE_12_HOUR_FORMAT" SHOW="true">
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="SET_FIELD_DELIMITER" FIELD="CHECK" REQUIRED="true" NUM_ROW="20" SHOW="false">
|
||||
<DEFAULT>true</DEFAULT>
|
||||
@@ -378,6 +381,7 @@
|
||||
<TEMPLATE_PARAM SOURCE="self.FILENAME" TARGET="tBQBE.GS_LOCAL_FILE" />
|
||||
|
||||
<TEMPLATE_PARAM SOURCE="self.ROWSEPARATOR" TARGET="tBQOB.ROWSEPARATOR" />
|
||||
<TEMPLATE_PARAM SOURCE="self.USE_12_HOUR_FORMAT" TARGET="tBQOB.USE_12_HOUR_FORMAT" />
|
||||
|
||||
<TEMPLATE_PARAM SOURCE="self.FIELDSEPARATOR" TARGET="tBQOB.FIELDSEPARATOR" />
|
||||
<TEMPLATE_PARAM SOURCE="self.DROP" TARGET="tBQBE.DROP" />
|
||||
|
||||
@@ -69,3 +69,4 @@ GS_SERVICE_ACCOUNT_KEY.NAME=Service account key
|
||||
GS_ACCESS_TOKEN.NAME=OAuth Access Token
|
||||
JOBID.NAME=BigQuery JobId
|
||||
STATISTICS.NAME=Statistics of the main query
|
||||
USE_12_HOUR_FORMAT.NAME=Use 12-hour format for dynamic columns
|
||||
|
||||
@@ -13,6 +13,7 @@ imports="
|
||||
String cid = node.getUniqueName();
|
||||
List<IMetadataTable> metadatas = node.getMetadataList();
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
boolean useTwelveHourFormat = ("true").equals(ElementParameterParser.getValue(node, "__USE_12_HOUR_FORMAT__"));
|
||||
if ((metadatas!=null)&&(metadatas.size()>0)) {
|
||||
IMetadataTable metadata = metadatas.get(0);
|
||||
if (metadata!=null) {
|
||||
@@ -22,6 +23,10 @@ imports="
|
||||
if(metadata.isDynamicSchema()) {
|
||||
%>
|
||||
class BigQueryOutputHelper_<%=cid%> {
|
||||
private final boolean useTwelveHourFormat;
|
||||
public BigQueryOutputHelper_<%=cid%>(boolean useTwelveHourFormat) {
|
||||
this.useTwelveHourFormat = useTwelveHourFormat;
|
||||
}
|
||||
public void writeData(routines.system.Dynamic column, String[] row, int offset) {
|
||||
for (int i = 0; i < column.getColumnCount(); i++) {
|
||||
if (column.getColumnValue(i) != null) {
|
||||
@@ -52,24 +57,30 @@ imports="
|
||||
}
|
||||
}
|
||||
public String getPattern(String typeName) {
|
||||
String hourFormat;
|
||||
if(useTwelveHourFormat) {
|
||||
hourFormat = "hh";
|
||||
} else {
|
||||
hourFormat = "HH";
|
||||
}
|
||||
if(typeName == null) {
|
||||
return "yyyy-MM-dd hh:mm:ss.SSS";
|
||||
return "yyyy-MM-dd " + hourFormat + ":mm:ss.SSS";
|
||||
}
|
||||
switch(typeName.toUpperCase()) {
|
||||
case "TIMESTAMP":
|
||||
return "yyyy-MM-dd hh:mm:ss.SSSXXX";
|
||||
return "yyyy-MM-dd " + hourFormat + ":mm:ss.SSSXXX";
|
||||
case "DATE":
|
||||
return "yyyy-MM-dd";
|
||||
case "TIME":
|
||||
return "hh:mm:ss.SSS";
|
||||
return hourFormat + ":mm:ss.SSS";
|
||||
case "DATETIME":
|
||||
return "yyyy-MM-dd hh:mm:ss.SSS";
|
||||
return "yyyy-MM-dd " + hourFormat + ":mm:ss.SSS";
|
||||
default:
|
||||
return "yyyy-MM-dd hh:mm:ss.SSS";
|
||||
return "yyyy-MM-dd " + hourFormat + ":mm:ss.SSS";
|
||||
}
|
||||
}
|
||||
}
|
||||
BigQueryOutputHelper_<%=cid%> bigQueryHelper_<%=cid%> = new BigQueryOutputHelper_<%=cid%>();
|
||||
BigQueryOutputHelper_<%=cid%> bigQueryHelper_<%=cid%> = new BigQueryOutputHelper_<%=cid%>(<%=useTwelveHourFormat%>);
|
||||
<%
|
||||
}
|
||||
|
||||
|
||||
@@ -43,6 +43,9 @@
|
||||
</PARAMETERS>
|
||||
|
||||
<ADVANCED_PARAMETERS>
|
||||
<PARAMETER NAME="USE_12_HOUR_FORMAT" FIELD="CHECK" REQUIRED="true" NUM_ROW="20" REPOSITORY_VALUE="USE_12_HOUR_FORMAT" SHOW="false">
|
||||
<DEFAULT>false</DEFAULT>
|
||||
</PARAMETER>
|
||||
<PARAMETER NAME="ROWSEPARATOR" FIELD="TEXT" REQUIRED="true" NUM_ROW="30" REPOSITORY_VALUE="ROW_SEPARATOR" SHOW="false">
|
||||
<DEFAULT>"\n"</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
@@ -228,7 +228,7 @@
|
||||
<IMPORT NAME="bonita-client-6.5.2.jar" MODULE="bonita-client-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-client-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="bonita-common-6.5.2.jar" MODULE="bonita-common-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-common-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.talend.libraries/httpmime-4.2.5/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.2.5.jar" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.2.5" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="httpclient-4.5.13.jar" MODULE="httpclient-4.5.13.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.13" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="commons-codec-1.14.jar" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
@@ -241,7 +241,7 @@
|
||||
<IMPORT NAME="bonita-server-6.5.2.jar" MODULE="bonita-server-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-server-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="xpp3-1.1.6.jar" MODULE="xpp3-1.1.6.jar" MVN="mvn:org.talend.libraries/xpp3-1.1.6/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.xml/lib/xpp3-1.1.6.jar" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.talend.libraries/httpmime-4.2.5/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.2.5.jar" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.2.5" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="httpclient-4.5.13.jar" MODULE="httpclient-4.5.13.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.13" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="commons-logging-1.2.jar" MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
@@ -252,7 +252,7 @@
|
||||
<IMPORT NAME="bonita-client-7.2.4.jar" MODULE="bonita-client-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-client-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="bonita-common-7.2.4.jar" MODULE="bonita-common-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-common-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.talend.libraries/httpmime-4.2.5/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.2.5.jar" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.2.5" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="httpclient-4.5.13.jar" MODULE="httpclient-4.5.13.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.13" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="commons-logging-1.2.jar" MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
@@ -265,12 +265,12 @@
|
||||
<IMPORT NAME="bonita-server-7.2.4.jar" MODULE="bonita-server-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-server-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="xpp3-1.1.6.jar" MODULE="xpp3-1.1.6.jar" MVN="mvn:org.talend.libraries/xpp3-1.1.6/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.xml/lib/xpp3-1.1.6.jar" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.talend.libraries/httpmime-4.2.5/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.2.5.jar" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.2.5" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="httpclient-4.5.13.jar" MODULE="httpclient-4.5.13.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.13" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="commons-logging-1.2.jar" MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="commons-codec-1.14.jar" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="commons-io-2.1.jar" MODULE="commons-io-2.1.jar" MVN="mvn:org.talend.libraries/commons-io-2.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.1.jar" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="commons-io-2.8.0.jar" MODULE="commons-io-2.8.0.jar" MVN="mvn:commons-io/commons-io/2.8.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -275,7 +275,7 @@
|
||||
<IMPORT NAME="bonita-client-6.5.2.jar" MODULE="bonita-client-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-client-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="bonita-common-6.5.2.jar" MODULE="bonita-common-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-common-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.talend.libraries/httpmime-4.2.5/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.2.5.jar" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.2.5" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="httpclient-4.5.13.jar" MODULE="httpclient-4.5.13.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.13" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="commons-codec-1.14.jar" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
@@ -289,7 +289,7 @@
|
||||
<IMPORT NAME="bonita-server-6.5.2.jar" MODULE="bonita-server-6.5.2.jar" MVN="mvn:org.talend.libraries/bonita-server-6.5.2/6.0.0" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="xpp3-1.1.6.jar" MODULE="xpp3-1.1.6.jar" MVN="mvn:org.talend.libraries/xpp3-1.1.6/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.xml/lib/xpp3-1.1.6.jar" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.talend.libraries/httpmime-4.2.5/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.2.5.jar" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.2.5" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="httpclient-4.5.13.jar" MODULE="httpclient-4.5.13.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.13" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="commons-logging-1.2.jar" MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2" REQUIRED_IF="(DB_VERSION=='BONITA_652') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
@@ -300,7 +300,7 @@
|
||||
<IMPORT NAME="bonita-client-7.2.4.jar" MODULE="bonita-client-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-client-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="bonita-common-7.2.4.jar" MODULE="bonita-common-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-common-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.talend.libraries/httpmime-4.2.5/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.2.5.jar" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.2.5" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="httpclient-4.5.13.jar" MODULE="httpclient-4.5.13.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.13" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
<IMPORT NAME="commons-logging-1.2.jar" MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='HTTP_CLIENT')" />
|
||||
@@ -314,12 +314,12 @@
|
||||
<IMPORT NAME="bonita-server-7.2.4.jar" MODULE="bonita-server-7.2.4.jar" MVN="mvn:org.talend.libraries/bonita-server-7.2.4/6.3.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="xpp3-1.1.6.jar" MODULE="xpp3-1.1.6.jar" MVN="mvn:org.talend.libraries/xpp3-1.1.6/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.xml/lib/xpp3-1.1.6.jar" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.talend.libraries/httpmime-4.2.5/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.2.5.jar" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="httpmime-4.2.5.jar" MODULE="httpmime-4.2.5.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.2.5" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="httpclient-4.5.13.jar" MODULE="httpclient-4.5.13.jar" MVN="mvn:org.apache.httpcomponents/httpclient/4.5.13" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="httpcore-4.4.13.jar" MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="commons-logging-1.2.jar" MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="commons-codec-1.14.jar" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="commons-io-2.1.jar" MODULE="commons-io-2.1.jar" MVN="mvn:org.talend.libraries/commons-io-2.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.1.jar" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
<IMPORT NAME="commons-io-2.8.0.jar" MODULE="commons-io-2.8.0.jar" MVN="mvn:commons-io/commons-io/2.8.0" REQUIRED_IF="(DB_VERSION=='BONITA_724') AND (CLIENT_MODE=='JAVA_CLIENT')" />
|
||||
|
||||
<!-- Bonita Http Client -->
|
||||
<IMPORT NAME="commons-codec-1.14" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="CLIENT_MODE=='HTTP_CLIENT'" />
|
||||
|
||||
@@ -87,8 +87,7 @@
|
||||
MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="apache-http-mime"
|
||||
MODULE="httpmime-4.1.2.jar" MVN="mvn:org.talend.libraries/httpmime-4.1.2/6.0.0"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.1.2.jar"
|
||||
MODULE="httpmime-4.1.2.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.1.2"
|
||||
REQUIRED="true" />
|
||||
<IMPORT NAME="apache-commons-logging"
|
||||
MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2"
|
||||
@@ -101,8 +100,7 @@
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar"
|
||||
REQUIRED="true" BundleID="" />
|
||||
<IMPORT NAME="apache-commons-io"
|
||||
MODULE="commons-io-2.4.jar" MVN="mvn:commons-io/commons-io/2.4"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.4.jar"
|
||||
MODULE="commons-io-2.8.0.jar" MVN="mvn:commons-io/commons-io/2.8.0"
|
||||
REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -142,8 +142,7 @@
|
||||
MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-mime"
|
||||
MODULE="httpmime-4.1.2.jar" MVN="mvn:org.talend.libraries/httpmime-4.1.2/6.0.0"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.1.2.jar"
|
||||
MODULE="httpmime-4.1.2.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.1.2"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-commons-logging"
|
||||
MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2"
|
||||
@@ -156,8 +155,7 @@
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" BundleID="" />
|
||||
<IMPORT NAME="apache-commons-io"
|
||||
MODULE="commons-io-2.4.jar" MVN="mvn:commons-io/commons-io/2.4"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.4.jar"
|
||||
MODULE="commons-io-2.8.0.jar" MVN="mvn:commons-io/commons-io/2.8.0"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -103,8 +103,7 @@
|
||||
MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-mime"
|
||||
MODULE="httpmime-4.1.2.jar" MVN="mvn:org.talend.libraries/httpmime-4.1.2/6.0.0"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.1.2.jar"
|
||||
MODULE="httpmime-4.1.2.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.1.2"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-commons-logging"
|
||||
MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2"
|
||||
@@ -117,8 +116,7 @@
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" BundleID="" />
|
||||
<IMPORT NAME="apache-commons-io"
|
||||
MODULE="commons-io-2.4.jar" MVN="mvn:commons-io/commons-io/2.4"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.4.jar"
|
||||
MODULE="commons-io-2.8.0.jar" MVN="mvn:commons-io/commons-io/2.8.0"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -113,8 +113,7 @@
|
||||
MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-mime"
|
||||
MODULE="httpmime-4.1.2.jar" MVN="mvn:org.talend.libraries/httpmime-4.1.2/6.0.0"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.1.2.jar"
|
||||
MODULE="httpmime-4.1.2.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.1.2"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-commons-logging"
|
||||
MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2"
|
||||
@@ -127,8 +126,7 @@
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" BundleID="" />
|
||||
<IMPORT NAME="apache-commons-io"
|
||||
MODULE="commons-io-2.4.jar" MVN="mvn:commons-io/commons-io/2.4"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.4.jar"
|
||||
MODULE="commons-io-2.8.0.jar" MVN="mvn:commons-io/commons-io/2.8.0"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -121,8 +121,7 @@
|
||||
MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-mime"
|
||||
MODULE="httpmime-4.1.2.jar" MVN="mvn:org.talend.libraries/httpmime-4.1.2/6.0.0"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.1.2.jar"
|
||||
MODULE="httpmime-4.1.2.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.1.2"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-commons-logging"
|
||||
MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2"
|
||||
@@ -135,8 +134,7 @@
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" BundleID="" />
|
||||
<IMPORT NAME="apache-commons-io"
|
||||
MODULE="commons-io-2.4.jar" MVN="mvn:commons-io/commons-io/2.4"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.4.jar"
|
||||
MODULE="commons-io-2.8.0.jar" MVN="mvn:commons-io/commons-io/2.8.0"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -124,8 +124,7 @@
|
||||
MODULE="httpcore-4.4.13.jar" MVN="mvn:org.apache.httpcomponents/httpcore/4.4.13"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-http-mime"
|
||||
MODULE="httpmime-4.1.2.jar" MVN="mvn:org.talend.libraries/httpmime-4.1.2/6.0.0"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/httpmime-4.1.2.jar"
|
||||
MODULE="httpmime-4.1.2.jar" MVN="mvn:org.apache.httpcomponents/httpmime/4.1.2"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
<IMPORT NAME="apache-commons-logging"
|
||||
MODULE="commons-logging-1.2.jar" MVN="mvn:commons-logging/commons-logging/1.2"
|
||||
@@ -138,8 +137,7 @@
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-lang-2.6.jar"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" BundleID="" />
|
||||
<IMPORT NAME="apache-commons-io"
|
||||
MODULE="commons-io-2.4.jar" MVN="mvn:commons-io/commons-io/2.4"
|
||||
UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.4.jar"
|
||||
MODULE="commons-io-2.8.0.jar" MVN="mvn:commons-io/commons-io/2.8.0"
|
||||
REQUIRED_IF="USE_EXISTING_CONNECTION == 'false'" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -165,7 +165,7 @@ COMPATIBILITY="ALL"
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="aopalliance-1_0" MODULE="aopalliance-1.0.jar" MVN="mvn:aopalliance/aopalliance/1.0" UrlPath="platform:/plugin/org.talend.libraries.esb/lib/aopalliance-1.0.jar" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="aws-ec2-2_0_2" MODULE="aws-ec2-2.0.2.jar" MVN="mvn:org.talend.libraries/aws-ec2-2.0.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="commons-io-2_4" MODULE="commons-io-2.4.jar" MVN="mvn:commons-io/commons-io/2.4" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.4.jar" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="commons-io-2.8.0" MODULE="commons-io-2.8.0.jar" MVN="mvn:commons-io/commons-io/2.8.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="ec2-2_0_2" MODULE="ec2-2.0.2.jar" MVN="mvn:org.talend.libraries/ec2-2.0.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="gson-2_2" MODULE="gson-2.2.jar" MVN="mvn:org.talend.libraries/gson-2.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="guava-16.0.1" MODULE="guava-16.0.1.jar" MVN="mvn:com.google.guava/guava/16.0.1" UrlPath="platform:/plugin/org.talend.libraries.guava/lib/guava-16.0.1.jar" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
|
||||
@@ -74,7 +74,7 @@ COMPATIBILITY="ALL"
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="aopalliance-1_0" MODULE="aopalliance-1.0.jar" MVN="mvn:aopalliance/aopalliance/1.0" UrlPath="platform:/plugin/org.talend.libraries.esb/lib/aopalliance-1.0.jar" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="aws-ec2-2_0_2" MODULE="aws-ec2-2.0.2.jar" MVN="mvn:org.talend.libraries/aws-ec2-2.0.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="commons-io-2_4" MODULE="commons-io-2.4.jar" MVN="mvn:commons-io/commons-io/2.4" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.4.jar" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="commons-io-2.8.0" MODULE="commons-io-2.8.0.jar" MVN="mvn:commons-io/commons-io/2.8.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="ec2-2_0_2" MODULE="ec2-2.0.2.jar" MVN="mvn:org.talend.libraries/ec2-2.0.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="gson-2_2" MODULE="gson-2.2.jar" MVN="mvn:org.talend.libraries/gson-2.2/6.0.0" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
<IMPORT NAME="guava-16.0.1" MODULE="guava-16.0.1.jar" MVN="mvn:com.google.guava/guava/16.0.1" UrlPath="platform:/plugin/org.talend.libraries.guava/lib/guava-16.0.1.jar" REQUIRED_IF="PROVIDER=='AWS_EC2'"/>
|
||||
|
||||
@@ -860,15 +860,16 @@
|
||||
<IMPORT NAME="Driver_MaxDB" MODULE="sapdbc.jar" MVN="mvn:org.talend.libraries/sapdbc/6.0.0" REQUIRED_IF="DBTYPE=='MAXDB'" />
|
||||
<IMPORT NAME="Driver-MSSQL" MODULE="jtds-1.3.1-patch-20190523.jar" MVN="mvn:net.sourceforge.jtds/jtds/1.3.1-patch-20190523" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='JTDS')" />
|
||||
<IMPORT NAME="Driver-MSSQL-prop" MODULE="mssql-jdbc.jar" MVN="mvn:org.talend.libraries/mssql-jdbc/6.0.0" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='MSSQL_PROP')" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.5.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.5" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='MSSQL_PROP') AND (MSSQL_ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.7.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.7" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='MSSQL_PROP') AND (MSSQL_ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="commons-lang3" MODULE="commons-lang3-3.10.jar" MVN="mvn:org.apache.commons/commons-lang3/3.10" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='MSSQL_PROP') AND (MSSQL_ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="commons-codec" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='MSSQL_PROP') AND (MSSQL_ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="gson" MODULE="gson-2.8.6.jar" MVN="mvn:com.google.code.gson/gson/2.8.6" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='MSSQL_PROP') AND (MSSQL_ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-6.5.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/6.5" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='MSSQL_PROP') AND (MSSQL_ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-9.7.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/9.7" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='MSSQL_PROP') AND (MSSQL_ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="content-type" MODULE="content-type-2.1.jar" MVN="mvn:com.nimbusds/content-type/2.1" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='MSSQL_PROP') AND (MSSQL_ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="json-smart" MODULE="json-smart-2.4.7.jar" MVN="mvn:net.minidev/json-smart/2.4.7" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='MSSQL_PROP') AND (MSSQL_ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="nimbus-jose-jwt" MODULE="nimbus-jose-jwt-8.11.jar" MVN="mvn:com.nimbusds/nimbus-jose-jwt/8.11" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='MSSQL_PROP') AND (MSSQL_ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="javax.mail" MODULE="javax.mail-1.6.2.jar" MVN="mvn:com.sun.mail/javax.mail/1.6.2" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MSSQL') AND (MSSQL_DRIVER=='MSSQL_PROP') AND (MSSQL_ACTIVE_DIR_AUTH == 'true')" />
|
||||
<IMPORT NAME="Driver-MYSQL5" MODULE="mysql-connector-java-5.1.30-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.30-bin/6.0.0" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') and (DBTYPE=='MYSQL') and (DB_MYSQL_VERSION=='MYSQL_5')"/>
|
||||
<IMPORT NAME="Driver-MYSQL5" MODULE="mysql-connector-java-5.1.49.jar" MVN="mvn:mysql/mysql-connector-java/5.1.49" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') and (DBTYPE=='MYSQL') and (DB_MYSQL_VERSION=='MYSQL_5')"/>
|
||||
<IMPORT NAME="Driver-MYSQL8" MODULE="mysql-connector-java-8.0.18.jar" MVN="mvn:mysql/mysql-connector-java/8.0.18" REQUIRED_IF="(DB_MYSQL_VERSION == 'MYSQL_8') AND (USE_EXISTING_CONNECTION == 'false') AND (DBTYPE=='MYSQL')" />
|
||||
<IMPORT NAME="Driver-MARIADB" MODULE="mariadb-java-client-2.5.3.jar" MVN="mvn:org.mariadb.jdbc/mariadb-java-client/2.5.3" REQUIRED_IF="(USE_EXISTING_CONNECTION == 'false') and (DBTYPE=='MYSQL') and (DB_MYSQL_VERSION=='MARIADB')" />
|
||||
<IMPORT NAME="Driver-Oracle8i" MODULE="ojdbc12.jar" MVN="mvn:org.talend.libraries/ojdbc12/6.0.0" REQUIRED_IF="(DBTYPE=='DBORACLE') AND (DB_VERSION == 'ORACLE_8') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
|
||||
@@ -63,6 +63,7 @@ abstract class Manager {
|
||||
return additionalParams;
|
||||
}
|
||||
|
||||
//this is not good way, it make trouble if want to add jdbc parameters, that need a complex string append action, not good, we should avoid that
|
||||
public String difineConnParam(){
|
||||
StringBuilder javaCode = new StringBuilder();
|
||||
javaCode.append("String driverClass_"+cid+"=\"" + getDriver()+ "\";\r\n");
|
||||
@@ -2428,9 +2429,41 @@ if(columnList != null && columnList.size() > 0) {
|
||||
<%
|
||||
boolean useExistingConnection = "true".equals(ElementParameterParser.getValue(node,"__USE_EXISTING_CONNECTION__"));
|
||||
if(!useExistingConnection) {
|
||||
%>
|
||||
if("MYSQL".equals(dbType)) {
|
||||
String dbproperties = ElementParameterParser.getValue(node,"__PROPERTIES__");
|
||||
String dbhost = ElementParameterParser.getValue(node, "__HOST__");
|
||||
String dbport = ElementParameterParser.getValue(node, "__PORT__");
|
||||
String dbname = ElementParameterParser.getValue(node, "__DBNAME__");
|
||||
String userName = ElementParameterParser.getValue(node, "__USER__");
|
||||
|
||||
String dbVersion = ElementParameterParser.getValue(node, "__DB_MYSQL_VERSION__");
|
||||
|
||||
String jdbcURL = "jdbc:mysql";
|
||||
if("MARIADB".equals(dbVersion)){
|
||||
jdbcURL = "jdbc:mariadb";
|
||||
}
|
||||
|
||||
String driverClass = null;
|
||||
if("MARIADB".equals(dbVersion)){
|
||||
driverClass = "org.mariadb.jdbc.Driver";
|
||||
} else if ("MYSQL_8".equals(dbVersion)){
|
||||
driverClass = "com.mysql.cj.jdbc.Driver";
|
||||
} else {
|
||||
driverClass = "com.mysql.jdbc.Driver";
|
||||
}
|
||||
%>
|
||||
|
||||
String driverClass_<%=cid%> = "<%=driverClass%>";
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/Mysql/jdbcurl4basic.javajet"%>
|
||||
|
||||
String dbUser_<%=cid%> = <%=userName%>;
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
<%=manager.difineConnParam()%>
|
||||
<%
|
||||
}
|
||||
log4jCodeGenerateUtil.debugConnectionParams(node);
|
||||
//log4jCodeGenerateUtil.connect_begin();
|
||||
}
|
||||
|
||||
@@ -681,7 +681,7 @@
|
||||
<IMPORT NAME="Driver-DB2" MODULE="db2jcc4.jar" MVN="mvn:org.talend.libraries/db2jcc4/6.0.0" REQUIRED="true" />
|
||||
<IMPORT NAME="Driver-DB2" MODULE="db2jcc_license_cu.jar" MVN="mvn:org.talend.libraries/db2jcc_license_cu/6.0.0" REQUIRED="true" />
|
||||
<IMPORT NAME="Driver-DB2" MODULE="db2jcc_license_cisuz.jar" MVN="mvn:org.talend.libraries/db2jcc_license_cisuz/6.0.0" REQUIRED="true" />
|
||||
<IMPORT NAME="commons-io-2.4" MODULE="commons-io-2.4.jar" MVN="mvn:commons-io/commons-io/2.4" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-io-2.4.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="commons-io-2.8.0" MODULE="commons-io-2.8.0.jar" MVN="mvn:commons-io/commons-io/2.8.0" REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -194,11 +194,12 @@
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="Driver-jTDS" MODULE="jtds-1.3.1-patch-20190523.jar" MVN="mvn:net.sourceforge.jtds/jtds/1.3.1-patch-20190523" REQUIRED_IF="(DRIVER == 'JTDS') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="Driver-mssql-jdbc" MODULE="mssql-jdbc.jar" MVN="mvn:org.talend.libraries/mssql-jdbc/6.0.0" REQUIRED_IF="(DRIVER == 'MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.5.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.5" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (DRIVER=='MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="adal4j" MODULE="adal4j-1.6.7.jar" MVN="mvn:com.microsoft.azure/adal4j/1.6.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (DRIVER=='MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="commons-lang3" MODULE="commons-lang3-3.10.jar" MVN="mvn:org.apache.commons/commons-lang3/3.10" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (DRIVER=='MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="commons-codec" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (DRIVER=='MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="gson" MODULE="gson-2.8.6.jar" MVN="mvn:com.google.code.gson/gson/2.8.6" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (DRIVER=='MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-6.5.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/6.5" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (DRIVER=='MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="oauth2-oidc-sdk" MODULE="oauth2-oidc-sdk-9.7.jar" MVN="mvn:com.nimbusds/oauth2-oidc-sdk/9.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (DRIVER=='MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="content-type" MODULE="content-type-2.1.jar" MVN="mvn:com.nimbusds/content-type/2.1" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (DRIVER=='MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="json-smart" MODULE="json-smart-2.4.7.jar" MVN="mvn:net.minidev/json-smart/2.4.7" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (DRIVER=='MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="nimbus-jose-jwt" MODULE="nimbus-jose-jwt-8.11.jar" MVN="mvn:com.nimbusds/nimbus-jose-jwt/8.11" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (DRIVER=='MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="javax.mail" MODULE="javax.mail-1.6.2.jar" MVN="mvn:com.sun.mail/javax.mail/1.6.2" REQUIRED_IF="(ACTIVE_DIR_AUTH == 'true') AND (DRIVER=='MSSQL_PROP') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
|
||||
@@ -195,7 +195,7 @@
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="MYSQL_8" MODULE="mysql-connector-java-8.0.18.jar" MVN="mvn:mysql/mysql-connector-java/8.0.18" REQUIRED_IF="(DB_VERSION == 'MYSQL_8') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="MYSQL_5" MODULE="mysql-connector-java-5.1.30-bin.jar" MVN="mvn:org.talend.libraries/mysql-connector-java-5.1.30-bin/6.0.0" REQUIRED_IF="(DB_VERSION == 'MYSQL_5') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="MYSQL_5" MODULE="mysql-connector-java-5.1.49.jar" MVN="mvn:mysql/mysql-connector-java/5.1.49" REQUIRED_IF="(DB_VERSION == 'MYSQL_5') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
<IMPORT NAME="MARIADB" MODULE="mariadb-java-client-2.5.3.jar" MVN="mvn:org.mariadb.jdbc/mariadb-java-client/2.5.3" REQUIRED_IF="(DB_VERSION == 'MARIADB') AND (USE_EXISTING_CONNECTION == 'false')" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
@@ -100,11 +100,9 @@ imports="
|
||||
%>
|
||||
String driverClass_<%=cid%> = "<%=drverClass%>";
|
||||
java.lang.Class.forName(driverClass_<%=cid%>);
|
||||
<%if(dbproperties == null || ("\"\"").equals(dbproperties) || ("").equals(dbproperties)) {%>
|
||||
String url_<%=cid %> = "<%=jdbcURL%>://"+<%=dbhost%>+":"+<%=dbport%>+"/"+<%=dbname%>;
|
||||
<%} else {%>
|
||||
String url_<%=cid %> = "<%=jdbcURL%>://"+<%=dbhost%>+":"+<%=dbport%>+"/"+<%=dbname%>+"?"+<%=dbproperties%>;
|
||||
<%}%>
|
||||
|
||||
<%@ include file="@{org.talend.designer.components.localprovider}/components/templates/Mysql/jdbcurl4basic.javajet"%>
|
||||
|
||||
String dbUser_<%=cid%> = <%=dbuser%>;
|
||||
|
||||
<%
|
||||
|
||||
@@ -28,7 +28,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
boolean isLog4jEnabled = ("true").equals(ElementParameterParser.getValue(node.getProcess(), "__LOG4J_ACTIVATE__"));
|
||||
String cid = node.getUniqueName();
|
||||
|
||||
String dbtable = null;
|
||||
String tableName = null;
|
||||
String dbschema = ElementParameterParser.getValue(node,"__ELT_SCHEMA_NAME__");
|
||||
String uniqueNameConnection = null;
|
||||
INode previousNode = null;
|
||||
@@ -50,7 +50,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
IConnection connection = connections.get(0);
|
||||
previousNode = connection.getSource();
|
||||
String previousComponentName = previousNode.getUniqueName();
|
||||
dbtable = connection.getName();
|
||||
tableName = connection.getName();
|
||||
uniqueNameConnection = connection.getUniqueName();
|
||||
|
||||
%>
|
||||
@@ -58,17 +58,13 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
selectQueryColumnsName_<%=cid %> = (String) globalMap.get("<%=previousComponentName%>"+"QUERY_COLUMNS_NAME"+"<%=uniqueNameConnection%>");
|
||||
<%
|
||||
}
|
||||
String tableName=null;
|
||||
if(useDifferentTable){
|
||||
tableName=differenttable;
|
||||
}else{
|
||||
if(isTableNameVariable){
|
||||
tableName=dbtable;
|
||||
}else{
|
||||
tableName="\""+dbtable +"\"";
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
if(useDifferentTable) {
|
||||
tableName = differenttable;
|
||||
} else if (!isTableNameVariable) {
|
||||
tableName = "\"" + tableName + "\"";
|
||||
}%>
|
||||
|
||||
String dbschema_<%=cid%> = <%=dbschema%>;
|
||||
if(dbschema_<%=cid%> != null && dbschema_<%=cid%>.trim().length() > 0) {
|
||||
tableName_<%=cid%> = <%=dbschema%> + "." + <%=tableName%>;
|
||||
@@ -267,7 +263,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
<%
|
||||
}
|
||||
|
||||
if(dbtable != null && columnList != null){
|
||||
if(tableName != null && columnList != null){
|
||||
|
||||
{//use the block to limit the code scope for table action
|
||||
Manager manager = null;
|
||||
@@ -276,8 +272,7 @@ skeleton="../templates/db_output_bulk.skeleton"
|
||||
|
||||
List<Column> stmtStructure = getELTManager(dbmsId, cid, node).createColumnList(columnList, false, null, null);
|
||||
|
||||
String table = useDifferentTable ? differenttable : ("\""+dbtable +"\"");
|
||||
|
||||
String table = tableName;
|
||||
dbLog = new DBLogUtil(node);
|
||||
%>
|
||||
<%@ include file="../templates/_tableActionForELT.javajet"%>
|
||||
|
||||
@@ -10,12 +10,14 @@ COMPATIBILITY="ALL"
|
||||
RELEASE_DATE="2010012"
|
||||
STARTABLE="true"
|
||||
LOG4J_ENABLED="true"
|
||||
|
||||
VISIBLE="false"
|
||||
>
|
||||
<SIGNATURE></SIGNATURE>
|
||||
</HEADER>
|
||||
|
||||
<FAMILIES>
|
||||
<FAMILY>Databases/DB Specifics/eXist</FAMILY>
|
||||
<FAMILY>Deprecated</FAMILY>
|
||||
</FAMILIES>
|
||||
|
||||
<DOCUMENTATION>
|
||||
|
||||
@@ -11,12 +11,14 @@ COMPATIBILITY="ALL"
|
||||
STARTABLE="true"
|
||||
PARTITIONING="AUTO"
|
||||
LOG4J_ENABLED="true"
|
||||
|
||||
VISIBLE="false"
|
||||
>
|
||||
<SIGNATURE></SIGNATURE>
|
||||
</HEADER>
|
||||
|
||||
<FAMILIES>
|
||||
<FAMILY>Databases/DB Specifics/eXist</FAMILY>
|
||||
<FAMILY>Deprecated</FAMILY>
|
||||
</FAMILIES>
|
||||
|
||||
<DOCUMENTATION>
|
||||
|
||||
@@ -11,12 +11,14 @@ COMPATIBILITY="ALL"
|
||||
STARTABLE="true"
|
||||
PARTITIONING="AUTO"
|
||||
LOG4J_ENABLED="true"
|
||||
|
||||
VISIBLE="false"
|
||||
>
|
||||
<SIGNATURE></SIGNATURE>
|
||||
</HEADER>
|
||||
|
||||
<FAMILIES>
|
||||
<FAMILY>Databases/DB Specifics/eXist</FAMILY>
|
||||
<FAMILY>Deprecated</FAMILY>
|
||||
</FAMILIES>
|
||||
|
||||
<DOCUMENTATION>
|
||||
|
||||
@@ -10,12 +10,14 @@ COMPATIBILITY="ALL"
|
||||
RELEASE_DATE="2010012"
|
||||
STARTABLE="true"
|
||||
LOG4J_ENABLED="true"
|
||||
|
||||
VISIBLE="false"
|
||||
>
|
||||
<SIGNATURE></SIGNATURE>
|
||||
</HEADER>
|
||||
|
||||
<FAMILIES>
|
||||
<FAMILY>Databases/DB Specifics/eXist</FAMILY>
|
||||
<FAMILY>Deprecated</FAMILY>
|
||||
</FAMILIES>
|
||||
|
||||
<DOCUMENTATION>
|
||||
|
||||
@@ -11,12 +11,14 @@ COMPATIBILITY="ALL"
|
||||
STARTABLE="true"
|
||||
PARTITIONING="AUTO"
|
||||
LOG4J_ENABLED="true"
|
||||
|
||||
VISIBLE="false"
|
||||
>
|
||||
<SIGNATURE></SIGNATURE>
|
||||
</HEADER>
|
||||
|
||||
<FAMILIES>
|
||||
<FAMILY>Databases/DB Specifics/eXist</FAMILY>
|
||||
<FAMILY>Deprecated</FAMILY>
|
||||
</FAMILIES>
|
||||
|
||||
<DOCUMENTATION>
|
||||
|
||||
@@ -10,12 +10,14 @@ COMPATIBILITY="ALL"
|
||||
RELEASE_DATE="2010012"
|
||||
STARTABLE="true"
|
||||
LOG4J_ENABLED="true"
|
||||
|
||||
VISIBLE="false"
|
||||
>
|
||||
<SIGNATURE></SIGNATURE>
|
||||
</HEADER>
|
||||
|
||||
<FAMILIES>
|
||||
<FAMILY>Databases/DB Specifics/eXist</FAMILY>
|
||||
<FAMILY>Deprecated</FAMILY>
|
||||
</FAMILIES>
|
||||
|
||||
<DOCUMENTATION>
|
||||
|
||||
@@ -10,12 +10,14 @@ COMPATIBILITY="ALL"
|
||||
RELEASE_DATE="2010012"
|
||||
STARTABLE="true"
|
||||
LOG4J_ENABLED="true"
|
||||
|
||||
VISIBLE="false"
|
||||
>
|
||||
<SIGNATURE></SIGNATURE>
|
||||
</HEADER>
|
||||
|
||||
<FAMILIES>
|
||||
<FAMILY>Databases/DB Specifics/eXist</FAMILY>
|
||||
<FAMILY>Deprecated</FAMILY>
|
||||
</FAMILIES>
|
||||
|
||||
<DOCUMENTATION>
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
session.disconnect();
|
||||
<% } else { %>
|
||||
org.apache.commons.net.ftp.FTPClient conn = (org.apache.commons.net.ftp.FTPClient) connObj;
|
||||
conn.logout();
|
||||
conn.disconnect();
|
||||
<% }
|
||||
%>
|
||||
|
||||
@@ -89,6 +89,7 @@
|
||||
REQUIRED="true"
|
||||
NUM_ROW="4"
|
||||
REPOSITORY_VALUE="PASSWORD"
|
||||
NOT_SHOW_IF="((SFTP=='true') AND (AUTH_METHOD == 'PUBLICKEY'))"
|
||||
>
|
||||
<DEFAULT>""</DEFAULT>
|
||||
</PARAMETER>
|
||||
|
||||
@@ -46,53 +46,30 @@
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
}else if (ftps) {// *** ftps *** //
|
||||
if (("false").equals(useExistingConn)) {
|
||||
if(isLog4jEnabled) {
|
||||
} else if (!("true").equals(useExistingConn)) { // *** ftp and ftps *** //
|
||||
if (!ftps && ("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%
|
||||
if(isLog4jEnabled) {
|
||||
%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%
|
||||
}
|
||||
try {
|
||||
<%
|
||||
}
|
||||
} else {// *** ftp *** //
|
||||
if(!("true").equals(useExistingConn)){
|
||||
if(("true").equals(ignoreFailureAtQuit)){
|
||||
%>
|
||||
try{
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
}catch(java.io.IOException ioe_<%=cid%>){
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
//ignore failure
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
}
|
||||
<%
|
||||
}else{
|
||||
%>
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid%>.logout();
|
||||
ftp_<%=cid%>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}
|
||||
if (!ftps && ("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
} catch (java.io.IOException ioe_<%=cid%>) { //ignore failure
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -149,7 +149,7 @@
|
||||
<%
|
||||
}
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPFile[] filesInDirectory_<%=cid%> = null;
|
||||
org.apache.commons.net.ftp.FTPFile[] filesInDirectory_<%=cid%> = null;
|
||||
boolean getFileNamesFailed_<%=cid%> = false;
|
||||
try {
|
||||
filesInDirectory_<%=cid%> = ftp_<%=cid %>.listFiles();
|
||||
@@ -165,6 +165,24 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e_<%=cid%>.getMessage());
|
||||
%>
|
||||
}
|
||||
|
||||
String[] listNamesInDirectory_<%=cid%> = null;
|
||||
try {
|
||||
listNamesInDirectory_<%=cid%> = ftp_<%=cid %>.listNames();
|
||||
if(filesInDirectory_<%=cid%> == null){
|
||||
getFileNamesFailed_<%=cid%> = true;
|
||||
}
|
||||
} catch(java.io.IOException e_<%=cid%>) {
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",e_<%=cid%>.getMessage());
|
||||
getFileNamesFailed_<%=cid%> = true;
|
||||
<%
|
||||
if(isLog4jEnabled) {
|
||||
%>
|
||||
log.warn("<%=cid%> - " + e_<%=cid%>.getMessage());
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
|
||||
if (getFileNamesFailed_<%=cid %>) {
|
||||
<%
|
||||
if (isLog4jEnabled) {
|
||||
@@ -179,6 +197,42 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e_<%=cid%>.getMessage());
|
||||
%>
|
||||
} else {
|
||||
<% /*search files matches to filemask and delete them*/ %>
|
||||
if( listNamesInDirectory_<%=cid%>!=null && (filesInDirectory_<%=cid%> == null || filesInDirectory_<%=cid%>.length == 0)){
|
||||
for (String fileName_<%=cid%> : listNamesInDirectory_<%=cid%>) {
|
||||
if (fileName_<%=cid%>.matches(mask<%=cid %>)) {
|
||||
boolean successDeleted_<%=cid %> = false;
|
||||
<%
|
||||
if("FILE".equals(targetType)){
|
||||
%>
|
||||
successDeleted_<%=cid %> = ftp_<%=cid %>.deleteFile(fileName_<%=cid%>);
|
||||
|
||||
<%
|
||||
}else{
|
||||
%>
|
||||
successDeleted_<%=cid %> = ftp_<%=cid %>.removeDirectory(fileName_<%=cid%>);
|
||||
<%}%>
|
||||
if (successDeleted_<%=cid %>) {
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File deleted.");
|
||||
nb_file_<%=cid%>++;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - <%=targetType%> '" + fileName_<%=cid%> + "' was deleted from the remote directory " + <%=remotedir %>);
|
||||
<%}%>
|
||||
} else {
|
||||
String responceFTPMessage = ftp_<%=cid %>.getReplyString();
|
||||
String errorMessageDeleteFail = "<%=cid%> - <%=targetType%> '" + fileName_<%=cid%> + "' was not deleted. Error message: " + responceFTPMessage;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error(errorMessageDeleteFail);
|
||||
<%} else { %>
|
||||
System.err.println(errorMessageDeleteFail);
|
||||
<%}%>
|
||||
|
||||
<%if (("true").equals(dieOnError)) { %>
|
||||
throw new RuntimeException(errorMessageDeleteFail);
|
||||
<% } %>
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
for (org.apache.commons.net.ftp.FTPFile file_<%=cid %> : filesInDirectory_<%=cid%>) {
|
||||
String fileName_<%=cid%> = file_<%=cid %>.getName();
|
||||
if (fileName_<%=cid%>.matches(mask<%=cid %>)) {
|
||||
@@ -239,6 +293,7 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e_<%=cid%>.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}// listNamesInDirectory_<%=cid%>!=null
|
||||
}
|
||||
|
||||
<% /*return to root dir for next filemask*/ %>
|
||||
|
||||
@@ -49,43 +49,30 @@ if (sftp) { // *** sftp *** //
|
||||
c_<%=cid%>.cd(<%=remoteDir %>);
|
||||
<%
|
||||
}
|
||||
} else if (ftps) { // *** ftps *** //
|
||||
if (!("true").equals(useExistingConn)) {
|
||||
} else if (!("true").equals(useExistingConn)) { // *** ftp and ftps *** //
|
||||
if (!ftps && ("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
try {
|
||||
<%
|
||||
}
|
||||
} else { // *** ftp *** //
|
||||
if (!("true").equals(useExistingConn)) {
|
||||
if (("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
try {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
} catch(java.io.IOException ioe_<%=cid%>) {
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
//ignore failure
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid%>.logout();
|
||||
ftp_<%=cid%>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}
|
||||
if (!ftps && ("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
} catch (java.io.IOException ioe_<%=cid%>) { //ignore failure
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
@@ -290,6 +290,13 @@ if (sftp) {// *** sftp *** //
|
||||
}
|
||||
|
||||
ftpFiles_<%=cid %> = ftp_<%=cid %>.listFiles();
|
||||
<%if(!dirFull){%>
|
||||
String[] nameLists_<%=cid %> = ftp_<%=cid %>.listNames();
|
||||
List<String> nameListsTemp_<%=cid %> = new java.util.ArrayList<>();
|
||||
<%}%>
|
||||
|
||||
|
||||
|
||||
<%
|
||||
if ("true".equals(useExistingConn) && !moveToCurrentDir) {
|
||||
%>
|
||||
@@ -301,31 +308,40 @@ if (sftp) {// *** sftp *** //
|
||||
<%/*prepare masks*/ %>
|
||||
for (String filemask_<%=cid %> : maskList_<%=cid %>) {
|
||||
java.util.regex.Pattern fileNamePattern_<%=cid %> = java.util.regex.Pattern.compile(filemask_<%=cid %>.replaceAll("\\.", "\\\\.").replaceAll("\\*", ".*"));
|
||||
|
||||
|
||||
<%if(!dirFull){%>
|
||||
if(nameLists_<%=cid %> != null){
|
||||
for (String ftpFile_<%=cid %> : nameLists_<%=cid %>) {
|
||||
if (fileNamePattern_<%=cid %>.matcher(ftpFile_<%=cid %>).matches()) {
|
||||
nameListsTemp_<%=cid %>.add(ftpFile_<%=cid %>);
|
||||
}
|
||||
}
|
||||
}
|
||||
<%}else{%>
|
||||
for (org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> : ftpFiles_<%=cid %>) {
|
||||
if (fileNamePattern_<%=cid %>.matcher(ftpFile_<%=cid %>.getName()).matches()) {
|
||||
String ftpFileName_<%=cid %> = ftpFile_<%=cid %>.getName();
|
||||
if (ftpFileName_<%=cid %>!=null && fileNamePattern_<%=cid %>.matcher(ftpFileName_<%=cid %>).matches()) {
|
||||
fileListTemp_<%=cid %>.add(ftpFile_<%=cid %>);
|
||||
}
|
||||
}
|
||||
<%}%>
|
||||
}
|
||||
|
||||
String currentFilePath_<%=cid%> = remotedir_<%=cid %>;
|
||||
if(!remotedir_<%=cid%>.endsWith("/")&&!remotedir_<%=cid%>.endsWith("\\")){
|
||||
currentFilePath_<%=cid%> += "/";
|
||||
}
|
||||
<%if(!dirFull){%>
|
||||
for (String ftpFile_<%=cid %> : nameListsTemp_<%=cid %>) {
|
||||
String currentFileName_<%=cid%> = ftpFile_<%=cid %>;
|
||||
|
||||
<%}else{%>
|
||||
|
||||
for (org.apache.commons.net.ftp.FTPFile ftpFile_<%=cid %> : fileListTemp_<%=cid %>) {
|
||||
String currentFileName_<%=cid%> = null;
|
||||
<%
|
||||
if (dirFull) {
|
||||
%>
|
||||
currentFileName_<%=cid%> = ftpFile_<%=cid %>.toString();
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
currentFileName_<%=cid%> = ftpFile_<%=cid %>.getName();
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
if(isLog4jEnabled) {
|
||||
%>
|
||||
@@ -335,8 +351,14 @@ if (sftp) {// *** sftp *** //
|
||||
%>
|
||||
<%/*set to globalMap*/ %>
|
||||
globalMap.put("<%=cid %>_CURRENT_FILE", currentFileName_<%=cid%>);
|
||||
<%if(!dirFull){%>
|
||||
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%> + currentFileName_<%=cid%>);
|
||||
|
||||
<%}else{%>
|
||||
globalMap.put("<%=cid %>_CURRENT_FILEPATH", currentFilePath_<%=cid%> + ftpFile_<%=cid %>.getName());
|
||||
<%}%>
|
||||
nb_file_<%=cid %>++;
|
||||
|
||||
<%
|
||||
} //end common code part
|
||||
%>
|
||||
@@ -42,52 +42,30 @@ if (sftp && !("true").equals(useExistingConn)) {// *** sftp *** //
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
<%
|
||||
} else if (ftps && !("true").equals(useExistingConn)) {
|
||||
if(isLog4jEnabled) {
|
||||
} else if (!("true").equals(useExistingConn)) { // *** ftp and ftps *** //
|
||||
if (!ftps && ("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
try {
|
||||
<%
|
||||
}
|
||||
%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%
|
||||
if(isLog4jEnabled) {
|
||||
%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%
|
||||
}
|
||||
} else {// *** ftp *** //
|
||||
|
||||
if (!("true").equals(useExistingConn)) {
|
||||
if (("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
try {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
} catch (java.io.IOException ioe_<%=cid%>) {
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
//ignore failure
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
}
|
||||
<%
|
||||
} else {
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%}%>
|
||||
ftp_<%=cid%>.logout();
|
||||
ftp_<%=cid%>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}
|
||||
if (!ftps && ("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
} catch (java.io.IOException ioe_<%=cid%>) { //ignore failure
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
@@ -95,4 +73,4 @@ globalMap.put("<%=cid %>_NB_FILE",nb_file_<%=cid%>);
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Listed files count: " + nb_file_<%=cid%> + ".");
|
||||
<%}%>
|
||||
<%}%>
|
||||
@@ -62,46 +62,30 @@ if (sftp && !bUseExistingConn) { // *** sftp *** //
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
<%
|
||||
} else if (ftps && !bUseExistingConn) {
|
||||
} else if (!bUseExistingConn) { // *** ftp and ftps *** //
|
||||
if (!ftps && ("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid%>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
try {
|
||||
<%
|
||||
} else {// *** ftp *** //
|
||||
if (!bUseExistingConn) {
|
||||
if (("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
try {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
} catch (java.io.IOException ioe_<%=cid%>) {
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
//ignore failure
|
||||
}
|
||||
<%} else {%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
%>
|
||||
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid%>.logout();
|
||||
ftp_<%=cid%>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}
|
||||
if (!ftps && ("true").equals(ignoreFailureAtQuit)) {
|
||||
%>
|
||||
} catch (java.io.IOException ioe_<%=cid%>) { //ignore failure
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
}
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
@@ -320,11 +320,41 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
public void getAllFiles(String remoteDirectory, String localDirectory)
|
||||
throws IllegalStateException, IOException, java.io.FileNotFoundException {
|
||||
|
||||
chdir(remoteDirectory);
|
||||
if(!chdir(remoteDirectory)){
|
||||
//change dir fail
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("Change dir fail. Skip Directory: " + remoteDirectory);
|
||||
<%}%>
|
||||
return;
|
||||
}
|
||||
String path = ftpClient.printWorkingDirectory();
|
||||
org.apache.commons.net.ftp.FTPFile[] ftpFiles = null;
|
||||
ftpFiles = ftpClient.listFiles();
|
||||
String[] nameLists = null;
|
||||
try {
|
||||
nameLists = ftpClient.listNames();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
if( nameLists!=null && (ftpFiles == null || ftpFiles.length == 0)){
|
||||
//if the file is folder, catch the FTPException and recur
|
||||
for (String ftpFileName : nameLists){
|
||||
try{
|
||||
downloadFile(localDirectory + "/" + ftpFileName,ftpFileName);
|
||||
} catch (IOException e) {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + e.getMessage());
|
||||
<%}%>
|
||||
java.io.File localFile = new java.io.File(localDirectory + "/" + ftpFileName);
|
||||
|
||||
if (!localFile.exists()) {
|
||||
localFile.mkdir();
|
||||
}
|
||||
getAllFiles(path + "/" + ftpFileName, localDirectory + "/" + ftpFileName);
|
||||
chdir(remoteDirectory);
|
||||
}
|
||||
}
|
||||
}else{
|
||||
for (org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
|
||||
|
||||
if (ftpFile.isDirectory()) {
|
||||
@@ -343,19 +373,15 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void getFiles(String remoteDirectory, String localDirectory, String maskStr)
|
||||
throws IllegalStateException, IOException, java.io.FileNotFoundException {
|
||||
|
||||
chdir(remoteDirectory);
|
||||
org.apache.commons.net.ftp.FTPFile[] ftpFiles = null;
|
||||
ftpFiles = ftpClient.listFiles(".");
|
||||
|
||||
for(org.apache.commons.net.ftp.FTPFile ftpFile : ftpFiles) {
|
||||
|
||||
if (ftpFile.isFile()) {
|
||||
String fileName = ftpFile.getName();
|
||||
|
||||
String[] nameLists = ftpClient.listNames();
|
||||
if(nameLists != null){
|
||||
for (String fileName : nameLists) {
|
||||
if (fileName.matches(maskStr)) {
|
||||
downloadFile(localDirectory + "/" + fileName,fileName);
|
||||
}
|
||||
@@ -363,10 +389,11 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public void chdir(String path) throws IllegalStateException, IOException {
|
||||
public boolean chdir(String path) throws IllegalStateException, IOException {
|
||||
if (!".".equals(path)) {
|
||||
ftpClient.changeWorkingDirectory(path);
|
||||
return ftpClient.changeWorkingDirectory(path);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public String pwd() throws IllegalStateException, IOException {
|
||||
@@ -414,14 +441,15 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
final java.util.List<String> msg_<%=cid%> = new java.util.ArrayList<String>();
|
||||
FTPGetter_<%=cid%> getter_<%=cid%> = new FTPGetter_<%=cid%>() {
|
||||
@Override
|
||||
protected void downloadFile(String localFileName, String remoteFileName)
|
||||
protected void downloadFile(String localFileName, String remoteFileName)
|
||||
throws IllegalStateException, java.io.FileNotFoundException, IOException {
|
||||
java.io.File localFile = new java.io.File(localFileName);
|
||||
<%
|
||||
if (append) {
|
||||
%>
|
||||
try (java.io.FileOutputStream localFos = new java.io.FileOutputStream(localFileName, true)) {
|
||||
ftpClient.retrieveFile(remoteFileName, localFos);
|
||||
boolean status = ftpClient.retrieveFile(remoteFileName, localFos);
|
||||
if(status){
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
|
||||
<%}%>
|
||||
@@ -431,8 +459,12 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
|
||||
count++;
|
||||
}else{
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded unsuccessfully.");
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer fail.");
|
||||
}
|
||||
} catch (IOException e) {
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded unsuccessfully.");
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer fail.");
|
||||
throw e;
|
||||
@@ -441,7 +473,10 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
} else if ("never".equals(overwrite)) {
|
||||
%>
|
||||
if (!localFile.exists()) {
|
||||
downloadFileWithOverwrite(localFileName, remoteFileName);
|
||||
boolean status = downloadFileWithOverwrite(localFileName, remoteFileName);
|
||||
if(!status){
|
||||
localFile.delete();
|
||||
}
|
||||
} else {
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%= cid %> - file ["+ remoteFileName +"] exit transmission.");
|
||||
@@ -469,17 +504,21 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "No file transfered.");
|
||||
}
|
||||
} else {
|
||||
downloadFileWithOverwrite(localFileName, remoteFileName);
|
||||
boolean status = downloadFileWithOverwrite(localFileName, remoteFileName);
|
||||
if(!status){
|
||||
localFile.delete();
|
||||
}
|
||||
}
|
||||
<%
|
||||
}
|
||||
%>
|
||||
}
|
||||
|
||||
private void downloadFileWithOverwrite(String localFileName, String remoteFileName)
|
||||
private boolean downloadFileWithOverwrite(String localFileName, String remoteFileName)
|
||||
throws IllegalStateException, java.io.FileNotFoundException, IOException {
|
||||
try (java.io.FileOutputStream localFos = new java.io.FileOutputStream(localFileName)) {
|
||||
ftpClient.retrieveFile(remoteFileName, localFos);
|
||||
boolean status = ftpClient.retrieveFile(remoteFileName, localFos);
|
||||
if(status){
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
|
||||
<%}%>
|
||||
@@ -489,8 +528,10 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
<%}%>
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer OK.");
|
||||
count++;
|
||||
}
|
||||
return status;
|
||||
} catch (IOException e) {
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
msg_<%=cid%>.add("file [" + remoteFileName + "] downloaded unsuccessfully.");
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File transfer fail.");
|
||||
throw e;
|
||||
@@ -531,14 +572,16 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
%>
|
||||
FTPGetter_<%=cid%> getter_<%=cid%> = new FTPGetter_<%=cid%>() {
|
||||
@Override
|
||||
protected void downloadFile(String localFileName, String remoteFileName)
|
||||
protected void downloadFile(String localFileName, String remoteFileName)
|
||||
throws IllegalStateException, java.io.FileNotFoundException, IOException {
|
||||
try (java.io.FileOutputStream localFos = new java.io.FileOutputStream(localFileName)) {
|
||||
ftpClient.retrieveFile(remoteFileName, localFos);
|
||||
boolean status = ftpClient.retrieveFile(remoteFileName, localFos);
|
||||
if(status){
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.debug("<%=cid%> - Downloaded file " + (count +1) + " : '" + remoteFileName + "' successfully.");
|
||||
<%}%>
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -74,12 +74,13 @@ nb_file_<%=cid%> = getter_<%=cid %>.count;
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.logout();
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
} catch(java.io.IOException ioe_<%=cid%>) {
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
} catch(java.io.IOException ioe_<%=cid%>) {
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.error("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
@@ -91,7 +92,8 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
ftp_<%=cid %>.logout();
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
@@ -111,7 +113,8 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid%>.disconnect();
|
||||
ftp_<%=cid%>.logout();
|
||||
ftp_<%=cid%>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
|
||||
@@ -173,6 +173,9 @@ int nb_file_<%=cid%> = 0;
|
||||
String rootDir_<%=cid %> = c_<%=cid %>.pwd();
|
||||
String remotedir_<%=cid %> = (<%=remotedir %>).replaceAll("\\\\", "/");
|
||||
String[] dirsTree_<%=cid %> = remotedir_<%=cid %>.split("/");
|
||||
if (remotedir_<%=cid %>.startsWith("/")) {
|
||||
c_<%=cid %>.cd("/"); // go to root path if remote is absolute.
|
||||
}
|
||||
for (String dir: dirsTree_<%=cid %>) {
|
||||
if (dir.isEmpty()) continue;
|
||||
|
||||
@@ -312,9 +315,14 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e.getMessage());
|
||||
%>
|
||||
if(!cwdSuccess_<%=cid %>){
|
||||
String[] dirsTree_<%=cid %> = remotedir_<%=cid %>.split("/");
|
||||
if (remotedir_<%=cid %>.startsWith("/")) {
|
||||
ftp_<%=cid %>.changeWorkingDirectory("/"); // go to root path if remote is absolute.
|
||||
}
|
||||
for (String dir: dirsTree_<%=cid %>) {
|
||||
ftp_<%=cid %>.makeDirectory(dir);
|
||||
ftp_<%=cid %>.changeWorkingDirectory(dir);
|
||||
if (dir != null && !dir.isEmpty()) {
|
||||
ftp_<%=cid %>.makeDirectory(dir);
|
||||
ftp_<%=cid %>.changeWorkingDirectory(dir);
|
||||
}
|
||||
}
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
|
||||
cwdSuccess_<%=cid %> = ftp_<%=cid %>.changeWorkingDirectory(remotedir_<%=cid %>);
|
||||
|
||||
@@ -81,6 +81,7 @@ if(sftp){ // *** sftp *** //
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.logout();
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
@@ -98,6 +99,7 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.logout();
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
@@ -125,6 +127,7 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid%>.logout();
|
||||
ftp_<%=cid%>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
|
||||
@@ -48,63 +48,40 @@
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
} else if (ftps) { // *** ftps *** //
|
||||
} else {// *** ftp and ftps *** //
|
||||
%>
|
||||
} <% /* close main loop */ %>
|
||||
<%
|
||||
if (!moveToCurrentDir) {
|
||||
%>
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
if(!("true").equals(useExistingConn)){
|
||||
%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%
|
||||
if(isLog4jEnabled) {
|
||||
%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {// *** ftp *** //
|
||||
%>
|
||||
}
|
||||
} <% /* close main loop */ %>
|
||||
<%
|
||||
if(!("true").equals(useExistingConn)){
|
||||
if(("true").equals(ignoreFailureAtQuit)){
|
||||
if(!ftps && ("true").equals(ignoreFailureAtQuit)){
|
||||
%>
|
||||
try{
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.logout();
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
}catch(java.io.IOException ioe_<%=cid%>){
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
<%
|
||||
if(!ftps && ("true").equals(ignoreFailureAtQuit)){
|
||||
%>
|
||||
} catch(java.io.IOException ioe_<%=cid%>) { //ignore failure
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
//ignore failure
|
||||
}
|
||||
<%
|
||||
}else{
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
}else{
|
||||
if(!moveToCurrentDir){
|
||||
} else {
|
||||
if(!moveToCurrentDir){
|
||||
%>
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
|
||||
ftp_<%=cid %>.changeWorkingDirectory(rootDir_<%=cid %>);
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,57 +46,34 @@
|
||||
}
|
||||
%>
|
||||
<%
|
||||
} else if (ftps) { // *** ftps *** //
|
||||
} else { // *** ftp and ftps *** //
|
||||
%>
|
||||
} <% /* close main loop */ %>
|
||||
<%
|
||||
if (("false").equals(useExistingConn)) {
|
||||
if(isLog4jEnabled) {
|
||||
%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%
|
||||
}
|
||||
%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%
|
||||
if(isLog4jEnabled) {
|
||||
%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%
|
||||
}
|
||||
}
|
||||
} else {// *** ftp *** //
|
||||
%>
|
||||
} <% /* close main loop */ %>
|
||||
<%
|
||||
if(!("true").equals(useExistingConn)){
|
||||
if(("true").equals(ignoreFailureAtQuit)){
|
||||
if(!ftps && ("true").equals(ignoreFailureAtQuit)){
|
||||
%>
|
||||
try{
|
||||
<%
|
||||
}
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.logout();
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
}catch(java.io.IOException ioe_<%=cid%>){
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
<%
|
||||
if(!ftps && ("true").equals(ignoreFailureAtQuit)){
|
||||
%>
|
||||
} catch(java.io.IOException ioe_<%=cid%>) { //ignore failure
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",ioe_<%=cid%>.getMessage());
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.warn("<%=cid%> - " + ioe_<%=cid%>.getMessage());
|
||||
<%}%>
|
||||
//ignore failure
|
||||
}
|
||||
<%
|
||||
}else{
|
||||
%>
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Closing the connection to the server.");
|
||||
<%}%>
|
||||
ftp_<%=cid %>.disconnect();
|
||||
<%if(isLog4jEnabled){%>
|
||||
log.info("<%=cid%> - Connection to the server closed.");
|
||||
<%}%>
|
||||
<%
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,12 +129,15 @@
|
||||
<%
|
||||
}
|
||||
%>
|
||||
org.apache.commons.net.ftp.FTPFile[] filesInDirectory_<%=cid%> = null;
|
||||
String[] filesInDirectory_<%=cid%> = null;
|
||||
boolean getFileNamesFailed_<%=cid%> = false;
|
||||
try {
|
||||
filesInDirectory_<%=cid%> = ftp_<%=cid %>.listFiles();
|
||||
filesInDirectory_<%=cid%> = ftp_<%=cid %>.listNames();
|
||||
if(filesInDirectory_<%=cid%> == null){
|
||||
getFileNamesFailed_<%=cid%> = true;
|
||||
}
|
||||
} catch(java.io.IOException e_<%=cid%>) {
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",e_<%=cid%>.getMessage());
|
||||
globalMap.put("<%=cid%>_ERROR_MESSAGE",e_<%=cid%>.getMessage());
|
||||
getFileNamesFailed_<%=cid%> = true;
|
||||
<%
|
||||
if(isLog4jEnabled) {
|
||||
@@ -159,11 +162,9 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e_<%=cid%>.getMessage());
|
||||
%>
|
||||
} else {
|
||||
<% /*search files matches to filemask and truncate them*/ %>
|
||||
for (org.apache.commons.net.ftp.FTPFile file_<%=cid %> : filesInDirectory_<%=cid%>) {
|
||||
String fileName_<%=cid%> = file_<%=cid %>.getName();
|
||||
for (String fileName_<%=cid %> : filesInDirectory_<%=cid%>) {
|
||||
if (fileName_<%=cid%>.matches(mask<%=cid %>)) {
|
||||
boolean successTruncated_<%=cid %> = false;
|
||||
if (file_<%=cid %>.isFile()) {
|
||||
successTruncated_<%=cid %> = ftp_<%=cid %>.storeFile(fileName_<%=cid%>, new java.io.ByteArrayInputStream(new byte[]{}));
|
||||
if (successTruncated_<%=cid %>) {
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File truncate OK.");
|
||||
@@ -182,7 +183,6 @@ globalMap.put("<%=cid%>_ERROR_MESSAGE",e_<%=cid%>.getMessage());
|
||||
globalMap.put("<%=cid %>_CURRENT_STATUS", "File truncate fail.");
|
||||
throw new RuntimeException(errorMessageDeleteFail);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,10 +131,10 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="commons-compress-1.19" MODULE="commons-compress-1.19.jar" MVN="mvn:org.apache.commons/commons-compress/1.19" REQUIRED="true" />
|
||||
<IMPORT NAME="Encrypt-Zip" MODULE="checkArchive-1.1-20190917.jar" MVN="mvn:org.talend.libraries/checkArchive-1.1-20190917/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/checkArchive-1.1-20190917.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="zip4j-1.3.3.jar" MODULE="zip4j-1.3.3.jar" MVN="mvn:net.lingala.zip4j/zip4j/1.3.3" REQUIRED="true" />
|
||||
<IMPORT NAME="talendzip-1.1-20201120.jar" MODULE="talendzip-1.1-20201120.jar" MVN="mvn:org.talend.components/talendzip/1.1-20201120" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talendzip-1.1-20201120.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="commons-compress-1.21" MODULE="commons-compress-1.21.jar" MVN="mvn:org.apache.commons/commons-compress/1.21" REQUIRED="true" />
|
||||
<IMPORT NAME="Encrypt-Zip" MODULE="checkArchive-1.2-20210901.jar" MVN="mvn:org.talend.components/checkArchive/1.2-20210901" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/checkArchive-1.2-20210901.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="zip4j-1.3.3.jar" MODULE="zip4j-1.3.3.jar" MVN="mvn:net.lingala.zip4j/zip4j/1.3.3" REQUIRED="true" />
|
||||
<IMPORT NAME="talendzip-1.2-20210901.jar" MODULE="talendzip-1.2-20210901.jar" MVN="mvn:org.talend.components/talendzip/1.2-20210901" UrlPath="platform:/plugin/org.talend.libraries.custom/lib/talendzip-1.2-20210901.jar" REQUIRED="true" />
|
||||
</IMPORTS>
|
||||
</CODEGENERATION>
|
||||
|
||||
|
||||
@@ -427,7 +427,7 @@ COMPATIBILITY="ALL"
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="COMMONS-httpclient" MODULE="commons-httpclient-3.0.1.jar" MVN="mvn:commons-httpclient/commons-httpclient/3.0.1" UrlPath="platform:/plugin/org.talend.libraries.apache.http/lib/commons-httpclient-3.0.1.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="COMMONS-httpclient" MODULE="commons-httpclient-3.0.1.jar" MVN="mvn:commons-httpclient/commons-httpclient/3.0.1" REQUIRED="true" />
|
||||
<IMPORT NAME="COMMONS-Logging" MODULE="commons-logging-1.1.jar" MVN="mvn:org.talend.libraries/commons-logging-1.1/6.0.0" UrlPath="platform:/plugin/org.talend.libraries.apache.common/lib/commons-logging-1.1.jar" REQUIRED="true" />
|
||||
<IMPORT NAME="commons-codec-1.14" MODULE="commons-codec-1.14.jar" MVN="mvn:commons-codec/commons-codec/1.14" REQUIRED="true" />
|
||||
<IMPORT NAME="jcifs-samba" MODULE="jcifs-1.3.0.jar" MVN="mvn:org.talend.libraries/jcifs-1.3.0/6.0.0" REQUIRED="true" />
|
||||
|
||||
@@ -25,7 +25,7 @@ PRINT.NAME=Print response to console
|
||||
PROTO.ITEM.FTP_PROTO=ftp
|
||||
PROTO.ITEM.HTTP_PROTO=http
|
||||
PROTO.ITEM.HTTPS_PROTO=https
|
||||
PROTO.ITEM.SAMBA_PROTO=smb
|
||||
PROTO.ITEM.SAMBA_PROTO=smbv1 (Deprecated)
|
||||
PROTO.NAME=Procotol
|
||||
PROXY_HOST.NAME=Proxy host
|
||||
PROXY_PASSWORD.NAME=Proxy password
|
||||
|
||||
@@ -189,7 +189,7 @@
|
||||
|
||||
<IMPORT NAME="poi-ooxml" MODULE="poi-ooxml-4.1.2-20200903124306_modified_talend.jar" MVN="mvn:org.apache.poi/poi-ooxml/4.1.2-20200903124306_modified_talend" UrlPath="platform:/plugin/org.talend.libraries.excel/lib/poi-ooxml-4.1.2-20200903124306_modified_talend.jar" REQUIRED_IF="(VERSION_2007 == 'true')" />
|
||||
<IMPORT NAME="curvesapi" MODULE="curvesapi-1.06.jar" MVN="mvn:com.github.virtuald/curvesapi/1.06" REQUIRED_IF="(VERSION_2007 == 'true')" />
|
||||
<IMPORT NAME="commons-compress" MODULE="commons-compress-1.19.jar" MVN="mvn:org.apache.commons/commons-compress/1.19" REQUIRED_IF="(VERSION_2007 == 'true')" />
|
||||
<IMPORT NAME="commons-compress" MODULE="commons-compress-1.21.jar" MVN="mvn:org.apache.commons/commons-compress/1.21" REQUIRED_IF="(VERSION_2007 == 'true')" />
|
||||
|
||||
<IMPORT NAME="poi-ooxml-schemas" MODULE="poi-ooxml-schemas-4.1.2-20200903124306_modified_talend.jar" MVN="mvn:org.apache.poi/poi-ooxml-schemas/4.1.2-20200903124306_modified_talend" UrlPath="platform:/plugin/org.talend.libraries.excel/lib/poi-ooxml-schemas-4.1.2-20200903124306_modified_talend.jar" REQUIRED_IF="(VERSION_2007 == 'true')" />
|
||||
<IMPORT NAME="xmlbeans" MODULE="xmlbeans-3.1.0.jar" MVN="mvn:org.apache.xmlbeans/xmlbeans/3.1.0" REQUIRED_IF="(VERSION_2007 == 'true')" />
|
||||
|
||||
@@ -85,7 +85,7 @@
|
||||
|
||||
<CODEGENERATION>
|
||||
<IMPORTS>
|
||||
<IMPORT NAME="JavaMail" MODULE="mail.jar" MVN="mvn:org.talend.libraries/mail/6.0.0" REQUIRED_IF="(MAIL_TYPE == 'MIME')" />
|
||||
<IMPORT NAME="JavaMail" MODULE="javax.mail-1.6.2.jar" MVN="mvn:com.sun.mail/javax.mail/1.6.2" REQUIRED_IF="(MAIL_TYPE == 'MIME')" />
|
||||
<IMPORT NAME="JAF" MODULE="activation.jar" MVN="mvn:org.talend.libraries/activation/6.0.0" REQUIRED_IF="(MAIL_TYPE == 'MIME')" />
|
||||
|
||||
<IMPORT NAME="poi" MODULE="poi-4.1.2-20200903124306_modified_talend.jar" MVN="mvn:org.apache.poi/poi/4.1.2-20200903124306_modified_talend" UrlPath="platform:/plugin/org.talend.libraries.excel/lib/poi-4.1.2-20200903124306_modified_talend.jar" REQUIRED_IF="(MAIL_TYPE == 'MSG')" />
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user