Compare commits
12 Commits
master
...
igonchar/T
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d707168ad3 | ||
|
|
e7c77957b7 | ||
|
|
4224cf52fb | ||
|
|
1fdb52a548 | ||
|
|
6e3a39fb5a | ||
|
|
31d8d03d4b | ||
|
|
117ceeeeca | ||
|
|
1549c6a477 | ||
|
|
e70b585c7d | ||
|
|
9abd5693c0 | ||
|
|
a70a145945 | ||
|
|
d93e03058b |
@@ -69,7 +69,7 @@ if(hasInput){
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
boolean hasValidInput = inputConn!=null;
|
||||
if (hasValidInput) {
|
||||
List<IMetadataColumn> input_columnList = inputConn.getMetadataTable().getListColumns();
|
||||
@@ -120,6 +120,7 @@ if(hasInput){
|
||||
dm_<%=cid%>.getLogicalType(),
|
||||
dm_<%=cid%>.getFormat(),
|
||||
dm_<%=cid%>.getDescription(),
|
||||
dm_<%=cid%>.getDbType(),
|
||||
dm_<%=cid%>.isNullable());
|
||||
}
|
||||
incomingEnforcer_<%=cid%>.createRuntimeSchema();
|
||||
@@ -162,7 +163,7 @@ if(hasInput){
|
||||
// There will be a ClassCastException if the output component does
|
||||
// not implement WriterWithFeedback, but permits outgoing
|
||||
// connections.
|
||||
|
||||
|
||||
ComponentProperties componentProps = node.getComponentProperties();
|
||||
ProcessPropertiesGenerator generator = new ProcessPropertiesGenerator(cid, component);
|
||||
List<Component.CodegenPropInfo> propsToProcess = component.getCodegenPropInfos(componentProps);
|
||||
@@ -179,14 +180,14 @@ if(hasInput){
|
||||
}
|
||||
} // property
|
||||
} // propInfo
|
||||
|
||||
|
||||
%>
|
||||
|
||||
org.apache.avro.generic.IndexedRecord data_<%=cid%> = null;
|
||||
if (incomingEnforcer_<%=cid%> != null) {
|
||||
data_<%=cid%> = incomingEnforcer_<%=cid%>.getCurrentRecord();
|
||||
}
|
||||
|
||||
|
||||
<%
|
||||
boolean isParallelize ="true".equalsIgnoreCase(ElementParameterParser.getValue(node, "__PARALLELIZE__"));
|
||||
if (isParallelize) {
|
||||
@@ -201,7 +202,7 @@ if(hasInput){
|
||||
if (writer_<%=cid%> != null && data_<%=cid%> != null) {
|
||||
writer_<%=cid%>.write(data_<%=cid%>);
|
||||
}
|
||||
|
||||
|
||||
nb_line_<%=cid %>++;
|
||||
<%if(hasMainOutput){
|
||||
%>
|
||||
|
||||
@@ -7,9 +7,9 @@
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend-codegen-utils</artifactId>
|
||||
<!-- release for revert version of library -->
|
||||
<version>0.28.0</version>
|
||||
<version>0.28.2-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
|
||||
<properties>
|
||||
<avro.version>1.8.0</avro.version>
|
||||
<components.version>0.25.0-SNAPSHOT</components.version>
|
||||
@@ -19,13 +19,13 @@
|
||||
<java-formatter.plugin.version>0.1.0</java-formatter.plugin.version>
|
||||
<formatter.plugin.version>1.6.0-SNAPSHOT</formatter.plugin.version>
|
||||
<mockito.version>2.2.15</mockito.version>
|
||||
<jacoco.plugin.version>0.7.8</jacoco.plugin.version>
|
||||
<maven.compiler.source>1.7</maven.compiler.source>
|
||||
<maven.compiler.target>1.7</maven.compiler.target>
|
||||
<jacoco.plugin.version>0.8.5</jacoco.plugin.version>
|
||||
<maven.compiler.source>1.8</maven.compiler.source>
|
||||
<maven.compiler.target>1.8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
|
||||
</properties>
|
||||
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>talend_nexus</id>
|
||||
@@ -65,7 +65,7 @@
|
||||
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.talend.components</groupId>
|
||||
@@ -104,7 +104,7 @@
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
@@ -119,7 +119,7 @@
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
@@ -163,7 +163,7 @@
|
||||
<goal>validate</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
@@ -38,6 +38,8 @@ import org.talend.daikon.avro.AvroUtils;
|
||||
import org.talend.daikon.avro.LogicalTypeUtils;
|
||||
import org.talend.daikon.avro.SchemaConstants;
|
||||
|
||||
import static org.talend.codegen.DiSchemaConstants.TALEND6_COLUMN_TALEND_TYPE;
|
||||
|
||||
/**
|
||||
* <b>You should almost certainly not be using this class.</b>
|
||||
*
|
||||
@@ -60,11 +62,6 @@ public class IncomingSchemaEnforcer {
|
||||
*/
|
||||
private static final int NO_DYNAMIC_COLUMN = -1;
|
||||
|
||||
/**
|
||||
* The number of milliseconds in one day
|
||||
*/
|
||||
private static final long ONE_DAY = 1000 * 60 * 60 * 24;
|
||||
|
||||
/**
|
||||
* The design-time schema from the Studio that determines how incoming java column data will be interpreted.
|
||||
* This schema is retrieved from downstream component's properties
|
||||
@@ -156,6 +153,22 @@ public class IncomingSchemaEnforcer {
|
||||
*/
|
||||
public void addDynamicField(String name, String diType, String logicalType, String fieldPattern, String description,
|
||||
boolean isNullable) {
|
||||
addDynamicField(name, diType, logicalType, fieldPattern, description, null, isNullable);
|
||||
}
|
||||
|
||||
/**
|
||||
* Recreates dynamic field from parameters retrieved from DI dynamic metadata
|
||||
*
|
||||
* @param name dynamic field name
|
||||
* @param diType di column type
|
||||
* @param logicalType dynamic field logical type; could be null
|
||||
* @param fieldPattern dynamic field date format
|
||||
* @param description dynamic field description
|
||||
* @param dbType original database type used for this field in source data storage; may be null
|
||||
* @param isNullable defines whether dynamic field may contain <code>null</code> value
|
||||
*/
|
||||
public void addDynamicField(String name, String diType, String logicalType, String fieldPattern, String description,
|
||||
String dbType, boolean isNullable) {
|
||||
if (!needsInitDynamicColumns())
|
||||
return;
|
||||
Schema fieldSchema = diToAvro(diType, logicalType);
|
||||
@@ -168,11 +181,15 @@ public class IncomingSchemaEnforcer {
|
||||
if ("id_Date".equals(diType) && fieldPattern != null) {
|
||||
field.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, fieldPattern);
|
||||
}
|
||||
field.addProp(TALEND6_COLUMN_TALEND_TYPE, diType);
|
||||
if (dbType != null) {
|
||||
field.addProp(SchemaConstants.TALEND_COLUMN_DB_TYPE, dbType);
|
||||
}
|
||||
dynamicFields.add(field);
|
||||
}
|
||||
|
||||
public void addIncomingNodeField(String name, String className) {
|
||||
String diType = "id_String";
|
||||
String diType;
|
||||
switch (className) {
|
||||
case "java.lang.String":
|
||||
diType = "id_String";
|
||||
@@ -206,6 +223,7 @@ public class IncomingSchemaEnforcer {
|
||||
break;
|
||||
case "java.lang.Date":
|
||||
diType = "id_Date";
|
||||
break;
|
||||
default:
|
||||
diType = "id_String";
|
||||
}
|
||||
@@ -281,7 +299,7 @@ public class IncomingSchemaEnforcer {
|
||||
|
||||
// Copy all of the fields that were initialized from dynamic columns into the runtime Schema.
|
||||
boolean dynamicFieldsAdded = false;
|
||||
List<Schema.Field> fields = new ArrayList<Schema.Field>();
|
||||
List<Schema.Field> fields = new ArrayList<>();
|
||||
for (Schema.Field designField : designSchema.getFields()) {
|
||||
// Replace the dynamic column by all of its contents.
|
||||
if (designField.pos() == dynamicColumnPosition) {
|
||||
@@ -404,7 +422,7 @@ public class IncomingSchemaEnforcer {
|
||||
Object avroValue = null;
|
||||
|
||||
// TODO(rskraba): This is pretty rough -- fix with a general type conversion strategy.
|
||||
String talendType = field.getProp(DiSchemaConstants.TALEND6_COLUMN_TALEND_TYPE);
|
||||
String talendType = field.getProp(TALEND6_COLUMN_TALEND_TYPE);
|
||||
String javaClass = fieldSchema.getProp(SchemaConstants.JAVA_CLASS_FLAG);
|
||||
|
||||
// TODO(igonchar): This is wrong. However I left it as is. We have to fix it after release
|
||||
|
||||
@@ -22,11 +22,15 @@ import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertThat;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.talend.codegen.DiSchemaConstants.TALEND6_COLUMN_TALEND_TYPE;
|
||||
import static org.talend.daikon.avro.SchemaConstants.TALEND_COLUMN_DB_TYPE;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.avro.Schema;
|
||||
import org.apache.avro.SchemaBuilder;
|
||||
@@ -58,12 +62,12 @@ public class IncomingSchemaEnforcerTest {
|
||||
@Before
|
||||
public void setup() {
|
||||
Schema componentSchema = SchemaBuilder.builder().record("Record").fields() //
|
||||
.name("id").type().intType().noDefault() //
|
||||
.name("name").type().stringType().noDefault() //
|
||||
.name("age").type().intType().noDefault() //
|
||||
.name("valid").type().booleanType().noDefault() //
|
||||
.name("address").type().stringType().noDefault() //
|
||||
.name("comment").type().stringType().noDefault() //
|
||||
.name("id").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
|
||||
.name("name").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.name("age").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
|
||||
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
|
||||
.name("address").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.name("comment").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.endRecord();
|
||||
componentRecord = new GenericData.Record(componentSchema);
|
||||
componentRecord.put(0, 1);
|
||||
@@ -141,9 +145,9 @@ public class IncomingSchemaEnforcerTest {
|
||||
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0") //
|
||||
.prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true") //
|
||||
.fields() //
|
||||
.name("valid").type().booleanType().noDefault() //
|
||||
.name("address").type().stringType().noDefault() //
|
||||
.name("comment").type().stringType().noDefault() //
|
||||
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
|
||||
.name("address").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.name("comment").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.endRecord();
|
||||
|
||||
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
|
||||
@@ -186,9 +190,9 @@ public class IncomingSchemaEnforcerTest {
|
||||
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0") //
|
||||
.prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true") //
|
||||
.fields() //
|
||||
.name("valid").type().booleanType().noDefault() //
|
||||
.name("address").type().stringType().noDefault() //
|
||||
.name("comment").type().stringType().noDefault() //
|
||||
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
|
||||
.name("address").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.name("comment").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.endRecord();
|
||||
|
||||
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
|
||||
@@ -228,9 +232,9 @@ public class IncomingSchemaEnforcerTest {
|
||||
@Test
|
||||
public void testDynamicColumnDynamicColumnAtMiddleOld() {
|
||||
Schema designSchema = SchemaBuilder.builder().record("Record").fields() //
|
||||
.name("id").type().intType().noDefault() //
|
||||
.name("address").type().stringType().noDefault() //
|
||||
.name("comment").type().stringType().noDefault() //
|
||||
.name("id").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
|
||||
.name("address").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.name("comment").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.endRecord();
|
||||
designSchema = AvroUtils.setIncludeAllFields(designSchema, true);
|
||||
designSchema = AvroUtils.setProperty(designSchema, DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "1");
|
||||
@@ -275,9 +279,9 @@ public class IncomingSchemaEnforcerTest {
|
||||
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "1") //
|
||||
.prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true") //
|
||||
.fields() //
|
||||
.name("id").type().intType().noDefault() //
|
||||
.name("address").type().stringType().noDefault() //
|
||||
.name("comment").type().stringType().noDefault() //
|
||||
.name("id").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
|
||||
.name("address").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.name("comment").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.endRecord();
|
||||
|
||||
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
|
||||
@@ -318,9 +322,9 @@ public class IncomingSchemaEnforcerTest {
|
||||
public void testDynamicColumnDynamicColumnAtEndOld() {
|
||||
// The expected schema after enforcement.
|
||||
Schema talend6Schema = SchemaBuilder.builder().record("Record").fields() //
|
||||
.name("id").type().intType().noDefault() //
|
||||
.name("name").type().stringType().noDefault() //
|
||||
.name("age").type().intType().noDefault() //
|
||||
.name("id").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
|
||||
.name("name").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.name("age").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
|
||||
.endRecord();
|
||||
talend6Schema = AvroUtils.setIncludeAllFields(talend6Schema, true);
|
||||
talend6Schema = AvroUtils.setProperty(talend6Schema, DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "3");
|
||||
@@ -365,9 +369,9 @@ public class IncomingSchemaEnforcerTest {
|
||||
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "3") //
|
||||
.prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true") //
|
||||
.fields() //
|
||||
.name("id").type().intType().noDefault() //
|
||||
.name("name").type().stringType().noDefault() //
|
||||
.name("age").type().intType().noDefault() //
|
||||
.name("id").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
|
||||
.name("name").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.name("age").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
|
||||
.endRecord();
|
||||
|
||||
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
|
||||
@@ -400,7 +404,7 @@ public class IncomingSchemaEnforcerTest {
|
||||
.name("field")
|
||||
//
|
||||
// properties
|
||||
.prop(DiSchemaConstants.TALEND6_COLUMN_TALEND_TYPE, "id_Date")
|
||||
.prop(TALEND6_COLUMN_TALEND_TYPE, "id_Date")
|
||||
.prop(DiSchemaConstants.TALEND6_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'")
|
||||
// type
|
||||
.type().longType().noDefault() //
|
||||
@@ -457,7 +461,7 @@ public class IncomingSchemaEnforcerTest {
|
||||
@Test
|
||||
public void testDynamicColumnALLSupportedType() {
|
||||
Schema talend6Schema = SchemaBuilder.builder().record("Record").fields() //
|
||||
.name("valid").type().booleanType().noDefault() //
|
||||
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
|
||||
.endRecord();
|
||||
talend6Schema = AvroUtils.setIncludeAllFields(talend6Schema, true);
|
||||
talend6Schema = AvroUtils.setProperty(talend6Schema, DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0");
|
||||
@@ -568,22 +572,24 @@ public class IncomingSchemaEnforcerTest {
|
||||
public void testAddDynamicFieldAllTypes() {
|
||||
Schema expectedRuntimeSchema = SchemaBuilder.builder().record("Record") //
|
||||
.fields() //
|
||||
.name("Test_String").type().stringType().noDefault() //
|
||||
.name("Test_Boolean").type().booleanType().noDefault() //
|
||||
.name("Test_Integer").type().intType().noDefault() //
|
||||
.name("Test_Long").type().longType().noDefault() //
|
||||
.name("Test_Double").type().doubleType().noDefault() //
|
||||
.name("Test_Float").type().floatType().noDefault() //
|
||||
.name("Test_BigDecimal").type(AvroUtils._decimal()).noDefault() //
|
||||
.name("Test_String").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
|
||||
.name("Test_Boolean").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
|
||||
.name("Test_Integer").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
|
||||
.name("Test_Long").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Long").type().longType().noDefault() //
|
||||
.name("Test_Double").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Double").type().doubleType().noDefault() //
|
||||
.name("Test_Float").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Float").type().floatType().noDefault() //
|
||||
.name("Test_BigDecimal").prop(TALEND6_COLUMN_TALEND_TYPE, "id_BigDecimal").type(AvroUtils._decimal()).noDefault() //
|
||||
.name("Test_Date").prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'")
|
||||
.type(AvroUtils._date()).noDefault() //
|
||||
.name("Test_Byte").type(AvroUtils._byte()).noDefault() //
|
||||
.name("Test_Short").type(AvroUtils._short()).noDefault() //
|
||||
.name("Test_Character").type(AvroUtils._character()).noDefault() //
|
||||
.name("TestLogicalDate").type(AvroUtils._logicalDate()).noDefault() //
|
||||
.name("TestLogicalTimeMillis").type(AvroUtils._logicalTime()).noDefault() //
|
||||
.name("TestLogicalTimestampMillis").type(AvroUtils._logicalTimestamp()).noDefault() //
|
||||
.name("valid").type().booleanType().noDefault() //
|
||||
.prop(TALEND6_COLUMN_TALEND_TYPE, "id_Date").type(AvroUtils._date()).noDefault() //
|
||||
.name("Test_Byte").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Byte").type(AvroUtils._byte()).noDefault() //
|
||||
.name("Test_Short").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Short").type(AvroUtils._short()).noDefault() //
|
||||
.name("Test_Character").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Character").type(AvroUtils._character()).noDefault() //
|
||||
.name("TestLogicalDate").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Date").type(AvroUtils._logicalDate()).noDefault() //
|
||||
.name("TestLogicalTimeMillis").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Date").type(AvroUtils._logicalTime())
|
||||
.noDefault() //
|
||||
.name("TestLogicalTimestampMillis").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Date")
|
||||
.type(AvroUtils._logicalTimestamp()).noDefault() //
|
||||
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
|
||||
.endRecord(); //
|
||||
|
||||
Schema designSchema = SchemaBuilder.builder().record("Record") //
|
||||
@@ -591,7 +597,7 @@ public class IncomingSchemaEnforcerTest {
|
||||
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0") //
|
||||
.prop(DiSchemaConstants.TALEND6_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'") //
|
||||
.fields() //
|
||||
.name("valid").type().booleanType().noDefault() //
|
||||
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
|
||||
.endRecord();
|
||||
|
||||
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
|
||||
@@ -608,7 +614,7 @@ public class IncomingSchemaEnforcerTest {
|
||||
enforcer.addDynamicField("Test_Short", "id_Short", null, null, null, false);
|
||||
enforcer.addDynamicField("Test_Character", "id_Character", null, null, null, false);
|
||||
enforcer.addDynamicField("TestLogicalDate", "id_Date", "date", null, null, false);
|
||||
enforcer.addDynamicField("TestLogicalTimeMillis", "id_Integer", "time-millis", null, null, false);
|
||||
enforcer.addDynamicField("TestLogicalTimeMillis", "id_Date", "time-millis", null, null, false);
|
||||
enforcer.addDynamicField("TestLogicalTimestampMillis", "id_Date", "timestamp-millis", null, null, false);
|
||||
|
||||
enforcer.createRuntimeSchema();
|
||||
@@ -666,17 +672,18 @@ public class IncomingSchemaEnforcerTest {
|
||||
public void testAddDynamicFieldNullable() {
|
||||
Schema expectedRuntimeSchema = SchemaBuilder.builder().record("Record") //
|
||||
.fields() //
|
||||
.name("Test_BigDecimal").type(AvroUtils.wrapAsNullable(AvroUtils._decimal())).noDefault() //
|
||||
.name("Test_BigDecimal").prop(TALEND6_COLUMN_TALEND_TYPE, "id_BigDecimal")
|
||||
.type(AvroUtils.wrapAsNullable(AvroUtils._decimal())).noDefault() //
|
||||
.name("Test_Date").prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'")
|
||||
.type(AvroUtils.wrapAsNullable(AvroUtils._date())).noDefault() //
|
||||
.name("valid").type().booleanType().noDefault() //
|
||||
.prop(TALEND6_COLUMN_TALEND_TYPE, "id_Date").type(AvroUtils.wrapAsNullable(AvroUtils._date())).noDefault() //
|
||||
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
|
||||
.endRecord(); //
|
||||
|
||||
Schema designSchema = SchemaBuilder.builder().record("Record").prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true") //
|
||||
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0") //
|
||||
.prop(DiSchemaConstants.TALEND6_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'") //
|
||||
.fields() //
|
||||
.name("valid").type().booleanType().noDefault() //
|
||||
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
|
||||
.endRecord();
|
||||
|
||||
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
|
||||
@@ -699,6 +706,36 @@ public class IncomingSchemaEnforcerTest {
|
||||
assertThat(record.get(1), is((Object) new Date(1234567891011L)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks {@link IncomingSchemaEnforcer#addDynamicField(String, String, String, String, String, String, boolean)}
|
||||
* adds information about original type used in source database to runtime schema
|
||||
*/
|
||||
@Test
|
||||
public void testAddDynamicFieldDbTypePresent() {
|
||||
List<Schema.Field> fields = new ArrayList<>();
|
||||
Schema.Field field1 = new Schema.Field("StringField", Schema.create(Schema.Type.STRING), null, (Object) null);
|
||||
field1.addProp(TALEND6_COLUMN_TALEND_TYPE, "id_String");
|
||||
field1.addProp(TALEND_COLUMN_DB_TYPE, "TEXT");
|
||||
fields.add(field1);
|
||||
Schema.Field field2 = new Schema.Field("IntField", Schema.create(Schema.Type.INT), null, (Object) null);
|
||||
field2.addProp(TALEND6_COLUMN_TALEND_TYPE, "id_Integer");
|
||||
field2.addProp(TALEND_COLUMN_DB_TYPE, "INT32");
|
||||
fields.add(field2);
|
||||
|
||||
Schema expectedRuntimeSchema = Schema.createRecord("Record", null, null, false, fields);
|
||||
|
||||
Schema designSchema = SchemaBuilder.builder().record("Record").prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true")
|
||||
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0").fields().endRecord();
|
||||
|
||||
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
|
||||
enforcer.addDynamicField("StringField", "id_String", null, null, null, "TEXT", false);
|
||||
enforcer.addDynamicField("IntField", "id_Integer", null, null, null, "INT32", false);
|
||||
enforcer.createRuntimeSchema();
|
||||
|
||||
Schema actualRuntimeSchema = enforcer.getRuntimeSchema();
|
||||
assertEquals(expectedRuntimeSchema, actualRuntimeSchema);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks {@link IncomingSchemaEnforcer#put()} converts string value to date according pattern specified in dynamic field
|
||||
* TODO (iv.gonchar): this is incorrect behavior, because avro record should not contain java.util.Date value. It should store
|
||||
@@ -710,7 +747,7 @@ public class IncomingSchemaEnforcerTest {
|
||||
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0") //
|
||||
.prop(DiSchemaConstants.TALEND6_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'") //
|
||||
.fields() //
|
||||
.name("valid").type().booleanType().noDefault() //
|
||||
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
|
||||
.endRecord();
|
||||
|
||||
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
|
||||
@@ -736,7 +773,7 @@ public class IncomingSchemaEnforcerTest {
|
||||
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0") //
|
||||
.prop(DiSchemaConstants.TALEND6_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'") //
|
||||
.fields() //
|
||||
.name("valid").type().booleanType().noDefault() //
|
||||
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
|
||||
.endRecord();
|
||||
|
||||
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
|
||||
|
||||
@@ -1268,7 +1268,7 @@ public class Component extends AbstractBasicComponent {
|
||||
moduleNeeded = new ModuleNeeded(getName(), "", true, "mvn:org.slf4j/slf4j-log4j12/1.7.25");
|
||||
componentImportNeedsList.add(moduleNeeded);
|
||||
}
|
||||
moduleNeeded = new ModuleNeeded(getName(), "", true, "mvn:org.talend.libraries/talend-codegen-utils/0.28.0");
|
||||
moduleNeeded = new ModuleNeeded(getName(), "", true, "mvn:org.talend.libraries/talend-codegen-utils/0.28.2-SNAPSHOT");
|
||||
componentImportNeedsList.add(moduleNeeded);
|
||||
return componentImportNeedsList;
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
context="plugin:org.talend.libraries.custom"
|
||||
language="java"
|
||||
message="Library for components API"
|
||||
name="talend-codegen-utils.jar" mvn_uri="mvn:org.talend.libraries/talend-codegen-utils/0.28.0"
|
||||
name="talend-codegen-utils.jar" mvn_uri="mvn:org.talend.libraries/talend-codegen-utils/0.28.2-SNAPSHOT"
|
||||
required="true">
|
||||
</libraryNeeded>
|
||||
</extension>
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
<artifactItem>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
<artifactId>talend-codegen-utils</artifactId>
|
||||
<version>0.28.0</version>
|
||||
<version>0.28.2-SNAPSHOT</version>
|
||||
</artifactItem>
|
||||
<artifactItem>
|
||||
<groupId>org.talend.libraries</groupId>
|
||||
|
||||
Reference in New Issue
Block a user