Compare commits

...

12 Commits

Author SHA1 Message Date
Dmytro Grygorenko
d707168ad3 fix(TDI-43569): another tests iteration. 2021-01-05 09:50:49 +02:00
Dmytro Grygorenko
e7c77957b7 fix(TDI-43569): fixing formatter issue. 2020-12-30 16:28:52 +02:00
Dmytro Grygorenko
4224cf52fb fix(TDI-43569): preparations for TTPdev run 2020-12-30 16:11:51 +02:00
Ivan Gonchar
1fdb52a548 wip 2020-12-23 15:46:30 +02:00
Ivan Gonchar
6e3a39fb5a fix(TDI-42938): fix javadoc for addDynamicField() 2020-12-23 15:46:29 +02:00
Ivan Gonchar
31d8d03d4b fix(TDI-42938): use TALEND6_COLUMN_TALEND_TYPE constant instead of value 2020-12-23 15:46:28 +02:00
Ivan Gonchar
117ceeeeca fix(TDI-42938): remove redundant constant 2020-12-23 15:46:28 +02:00
Ivan Gonchar
1549c6a477 fix(TDI-42938): use TALEND6_COLUMN_TALEND_TYPE constant variable instead of value 2020-12-23 15:46:27 +02:00
Ivan Gonchar
e70b585c7d fix(TDI-42938): call addDynamicField() with dbType parameter 2020-12-23 15:46:20 +02:00
Ivan Gonchar
9abd5693c0 fix(TDI-42938): add dbType parameter to addDynamicField() method 2020-12-23 15:45:37 +02:00
Ivan Gonchar
a70a145945 fix(TDI-42938): modify tests according added DI type property 2020-12-23 15:45:37 +02:00
Ivan Gonchar
d93e03058b fix(TDI-42938): add talend type tag to runtime schema created by SchemaEnforcer 2020-12-23 15:45:36 +02:00
7 changed files with 131 additions and 75 deletions

View File

@@ -69,7 +69,7 @@ if(hasInput){
}
}
}
boolean hasValidInput = inputConn!=null;
if (hasValidInput) {
List<IMetadataColumn> input_columnList = inputConn.getMetadataTable().getListColumns();
@@ -120,6 +120,7 @@ if(hasInput){
dm_<%=cid%>.getLogicalType(),
dm_<%=cid%>.getFormat(),
dm_<%=cid%>.getDescription(),
dm_<%=cid%>.getDbType(),
dm_<%=cid%>.isNullable());
}
incomingEnforcer_<%=cid%>.createRuntimeSchema();
@@ -162,7 +163,7 @@ if(hasInput){
// There will be a ClassCastException if the output component does
// not implement WriterWithFeedback, but permits outgoing
// connections.
ComponentProperties componentProps = node.getComponentProperties();
ProcessPropertiesGenerator generator = new ProcessPropertiesGenerator(cid, component);
List<Component.CodegenPropInfo> propsToProcess = component.getCodegenPropInfos(componentProps);
@@ -179,14 +180,14 @@ if(hasInput){
}
} // property
} // propInfo
%>
org.apache.avro.generic.IndexedRecord data_<%=cid%> = null;
if (incomingEnforcer_<%=cid%> != null) {
data_<%=cid%> = incomingEnforcer_<%=cid%>.getCurrentRecord();
}
<%
boolean isParallelize ="true".equalsIgnoreCase(ElementParameterParser.getValue(node, "__PARALLELIZE__"));
if (isParallelize) {
@@ -201,7 +202,7 @@ if(hasInput){
if (writer_<%=cid%> != null && data_<%=cid%> != null) {
writer_<%=cid%>.write(data_<%=cid%>);
}
nb_line_<%=cid %>++;
<%if(hasMainOutput){
%>

View File

@@ -7,9 +7,9 @@
<groupId>org.talend.libraries</groupId>
<artifactId>talend-codegen-utils</artifactId>
<!-- release for revert version of library -->
<version>0.28.0</version>
<version>0.28.2-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
<avro.version>1.8.0</avro.version>
<components.version>0.25.0-SNAPSHOT</components.version>
@@ -19,13 +19,13 @@
<java-formatter.plugin.version>0.1.0</java-formatter.plugin.version>
<formatter.plugin.version>1.6.0-SNAPSHOT</formatter.plugin.version>
<mockito.version>2.2.15</mockito.version>
<jacoco.plugin.version>0.7.8</jacoco.plugin.version>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
<jacoco.plugin.version>0.8.5</jacoco.plugin.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<talend.nexus.url>https://artifacts-oss.talend.com</talend.nexus.url>
</properties>
<repositories>
<repository>
<id>talend_nexus</id>
@@ -65,7 +65,7 @@
<url>${talend.nexus.url}/nexus/content/repositories/TalendOpenSourceRelease/</url>
</pluginRepository>
</pluginRepositories>
<dependencies>
<dependency>
<groupId>org.talend.components</groupId>
@@ -104,7 +104,7 @@
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
@@ -119,7 +119,7 @@
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
@@ -163,7 +163,7 @@
<goal>validate</goal>
</goals>
</execution>
</executions>
</executions>
</plugin>
</plugins>
</build>

View File

@@ -38,6 +38,8 @@ import org.talend.daikon.avro.AvroUtils;
import org.talend.daikon.avro.LogicalTypeUtils;
import org.talend.daikon.avro.SchemaConstants;
import static org.talend.codegen.DiSchemaConstants.TALEND6_COLUMN_TALEND_TYPE;
/**
* <b>You should almost certainly not be using this class.</b>
*
@@ -60,11 +62,6 @@ public class IncomingSchemaEnforcer {
*/
private static final int NO_DYNAMIC_COLUMN = -1;
/**
* The number of milliseconds in one day
*/
private static final long ONE_DAY = 1000 * 60 * 60 * 24;
/**
* The design-time schema from the Studio that determines how incoming java column data will be interpreted.
* This schema is retrieved from downstream component's properties
@@ -156,6 +153,22 @@ public class IncomingSchemaEnforcer {
*/
public void addDynamicField(String name, String diType, String logicalType, String fieldPattern, String description,
boolean isNullable) {
addDynamicField(name, diType, logicalType, fieldPattern, description, null, isNullable);
}
/**
* Recreates dynamic field from parameters retrieved from DI dynamic metadata
*
* @param name dynamic field name
* @param diType di column type
* @param logicalType dynamic field logical type; could be null
* @param fieldPattern dynamic field date format
* @param description dynamic field description
* @param dbType original database type used for this field in source data storage; may be null
* @param isNullable defines whether dynamic field may contain <code>null</code> value
*/
public void addDynamicField(String name, String diType, String logicalType, String fieldPattern, String description,
String dbType, boolean isNullable) {
if (!needsInitDynamicColumns())
return;
Schema fieldSchema = diToAvro(diType, logicalType);
@@ -168,11 +181,15 @@ public class IncomingSchemaEnforcer {
if ("id_Date".equals(diType) && fieldPattern != null) {
field.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, fieldPattern);
}
field.addProp(TALEND6_COLUMN_TALEND_TYPE, diType);
if (dbType != null) {
field.addProp(SchemaConstants.TALEND_COLUMN_DB_TYPE, dbType);
}
dynamicFields.add(field);
}
public void addIncomingNodeField(String name, String className) {
String diType = "id_String";
String diType;
switch (className) {
case "java.lang.String":
diType = "id_String";
@@ -206,6 +223,7 @@ public class IncomingSchemaEnforcer {
break;
case "java.lang.Date":
diType = "id_Date";
break;
default:
diType = "id_String";
}
@@ -281,7 +299,7 @@ public class IncomingSchemaEnforcer {
// Copy all of the fields that were initialized from dynamic columns into the runtime Schema.
boolean dynamicFieldsAdded = false;
List<Schema.Field> fields = new ArrayList<Schema.Field>();
List<Schema.Field> fields = new ArrayList<>();
for (Schema.Field designField : designSchema.getFields()) {
// Replace the dynamic column by all of its contents.
if (designField.pos() == dynamicColumnPosition) {
@@ -404,7 +422,7 @@ public class IncomingSchemaEnforcer {
Object avroValue = null;
// TODO(rskraba): This is pretty rough -- fix with a general type conversion strategy.
String talendType = field.getProp(DiSchemaConstants.TALEND6_COLUMN_TALEND_TYPE);
String talendType = field.getProp(TALEND6_COLUMN_TALEND_TYPE);
String javaClass = fieldSchema.getProp(SchemaConstants.JAVA_CLASS_FLAG);
// TODO(igonchar): This is wrong. However I left it as is. We have to fix it after release

View File

@@ -22,11 +22,15 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.talend.codegen.DiSchemaConstants.TALEND6_COLUMN_TALEND_TYPE;
import static org.talend.daikon.avro.SchemaConstants.TALEND_COLUMN_DB_TYPE;
import java.math.BigDecimal;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.avro.Schema;
import org.apache.avro.SchemaBuilder;
@@ -58,12 +62,12 @@ public class IncomingSchemaEnforcerTest {
@Before
public void setup() {
Schema componentSchema = SchemaBuilder.builder().record("Record").fields() //
.name("id").type().intType().noDefault() //
.name("name").type().stringType().noDefault() //
.name("age").type().intType().noDefault() //
.name("valid").type().booleanType().noDefault() //
.name("address").type().stringType().noDefault() //
.name("comment").type().stringType().noDefault() //
.name("id").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
.name("name").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.name("age").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
.name("address").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.name("comment").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.endRecord();
componentRecord = new GenericData.Record(componentSchema);
componentRecord.put(0, 1);
@@ -141,9 +145,9 @@ public class IncomingSchemaEnforcerTest {
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0") //
.prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true") //
.fields() //
.name("valid").type().booleanType().noDefault() //
.name("address").type().stringType().noDefault() //
.name("comment").type().stringType().noDefault() //
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
.name("address").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.name("comment").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.endRecord();
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
@@ -186,9 +190,9 @@ public class IncomingSchemaEnforcerTest {
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0") //
.prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true") //
.fields() //
.name("valid").type().booleanType().noDefault() //
.name("address").type().stringType().noDefault() //
.name("comment").type().stringType().noDefault() //
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
.name("address").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.name("comment").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.endRecord();
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
@@ -228,9 +232,9 @@ public class IncomingSchemaEnforcerTest {
@Test
public void testDynamicColumnDynamicColumnAtMiddleOld() {
Schema designSchema = SchemaBuilder.builder().record("Record").fields() //
.name("id").type().intType().noDefault() //
.name("address").type().stringType().noDefault() //
.name("comment").type().stringType().noDefault() //
.name("id").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
.name("address").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.name("comment").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.endRecord();
designSchema = AvroUtils.setIncludeAllFields(designSchema, true);
designSchema = AvroUtils.setProperty(designSchema, DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "1");
@@ -275,9 +279,9 @@ public class IncomingSchemaEnforcerTest {
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "1") //
.prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true") //
.fields() //
.name("id").type().intType().noDefault() //
.name("address").type().stringType().noDefault() //
.name("comment").type().stringType().noDefault() //
.name("id").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
.name("address").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.name("comment").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.endRecord();
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
@@ -318,9 +322,9 @@ public class IncomingSchemaEnforcerTest {
public void testDynamicColumnDynamicColumnAtEndOld() {
// The expected schema after enforcement.
Schema talend6Schema = SchemaBuilder.builder().record("Record").fields() //
.name("id").type().intType().noDefault() //
.name("name").type().stringType().noDefault() //
.name("age").type().intType().noDefault() //
.name("id").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
.name("name").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.name("age").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
.endRecord();
talend6Schema = AvroUtils.setIncludeAllFields(talend6Schema, true);
talend6Schema = AvroUtils.setProperty(talend6Schema, DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "3");
@@ -365,9 +369,9 @@ public class IncomingSchemaEnforcerTest {
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "3") //
.prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true") //
.fields() //
.name("id").type().intType().noDefault() //
.name("name").type().stringType().noDefault() //
.name("age").type().intType().noDefault() //
.name("id").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
.name("name").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.name("age").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
.endRecord();
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
@@ -400,7 +404,7 @@ public class IncomingSchemaEnforcerTest {
.name("field")
//
// properties
.prop(DiSchemaConstants.TALEND6_COLUMN_TALEND_TYPE, "id_Date")
.prop(TALEND6_COLUMN_TALEND_TYPE, "id_Date")
.prop(DiSchemaConstants.TALEND6_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'")
// type
.type().longType().noDefault() //
@@ -457,7 +461,7 @@ public class IncomingSchemaEnforcerTest {
@Test
public void testDynamicColumnALLSupportedType() {
Schema talend6Schema = SchemaBuilder.builder().record("Record").fields() //
.name("valid").type().booleanType().noDefault() //
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
.endRecord();
talend6Schema = AvroUtils.setIncludeAllFields(talend6Schema, true);
talend6Schema = AvroUtils.setProperty(talend6Schema, DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0");
@@ -568,22 +572,24 @@ public class IncomingSchemaEnforcerTest {
public void testAddDynamicFieldAllTypes() {
Schema expectedRuntimeSchema = SchemaBuilder.builder().record("Record") //
.fields() //
.name("Test_String").type().stringType().noDefault() //
.name("Test_Boolean").type().booleanType().noDefault() //
.name("Test_Integer").type().intType().noDefault() //
.name("Test_Long").type().longType().noDefault() //
.name("Test_Double").type().doubleType().noDefault() //
.name("Test_Float").type().floatType().noDefault() //
.name("Test_BigDecimal").type(AvroUtils._decimal()).noDefault() //
.name("Test_String").prop(TALEND6_COLUMN_TALEND_TYPE, "id_String").type().stringType().noDefault() //
.name("Test_Boolean").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
.name("Test_Integer").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Integer").type().intType().noDefault() //
.name("Test_Long").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Long").type().longType().noDefault() //
.name("Test_Double").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Double").type().doubleType().noDefault() //
.name("Test_Float").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Float").type().floatType().noDefault() //
.name("Test_BigDecimal").prop(TALEND6_COLUMN_TALEND_TYPE, "id_BigDecimal").type(AvroUtils._decimal()).noDefault() //
.name("Test_Date").prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'")
.type(AvroUtils._date()).noDefault() //
.name("Test_Byte").type(AvroUtils._byte()).noDefault() //
.name("Test_Short").type(AvroUtils._short()).noDefault() //
.name("Test_Character").type(AvroUtils._character()).noDefault() //
.name("TestLogicalDate").type(AvroUtils._logicalDate()).noDefault() //
.name("TestLogicalTimeMillis").type(AvroUtils._logicalTime()).noDefault() //
.name("TestLogicalTimestampMillis").type(AvroUtils._logicalTimestamp()).noDefault() //
.name("valid").type().booleanType().noDefault() //
.prop(TALEND6_COLUMN_TALEND_TYPE, "id_Date").type(AvroUtils._date()).noDefault() //
.name("Test_Byte").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Byte").type(AvroUtils._byte()).noDefault() //
.name("Test_Short").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Short").type(AvroUtils._short()).noDefault() //
.name("Test_Character").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Character").type(AvroUtils._character()).noDefault() //
.name("TestLogicalDate").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Date").type(AvroUtils._logicalDate()).noDefault() //
.name("TestLogicalTimeMillis").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Date").type(AvroUtils._logicalTime())
.noDefault() //
.name("TestLogicalTimestampMillis").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Date")
.type(AvroUtils._logicalTimestamp()).noDefault() //
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
.endRecord(); //
Schema designSchema = SchemaBuilder.builder().record("Record") //
@@ -591,7 +597,7 @@ public class IncomingSchemaEnforcerTest {
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0") //
.prop(DiSchemaConstants.TALEND6_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'") //
.fields() //
.name("valid").type().booleanType().noDefault() //
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
.endRecord();
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
@@ -608,7 +614,7 @@ public class IncomingSchemaEnforcerTest {
enforcer.addDynamicField("Test_Short", "id_Short", null, null, null, false);
enforcer.addDynamicField("Test_Character", "id_Character", null, null, null, false);
enforcer.addDynamicField("TestLogicalDate", "id_Date", "date", null, null, false);
enforcer.addDynamicField("TestLogicalTimeMillis", "id_Integer", "time-millis", null, null, false);
enforcer.addDynamicField("TestLogicalTimeMillis", "id_Date", "time-millis", null, null, false);
enforcer.addDynamicField("TestLogicalTimestampMillis", "id_Date", "timestamp-millis", null, null, false);
enforcer.createRuntimeSchema();
@@ -666,17 +672,18 @@ public class IncomingSchemaEnforcerTest {
public void testAddDynamicFieldNullable() {
Schema expectedRuntimeSchema = SchemaBuilder.builder().record("Record") //
.fields() //
.name("Test_BigDecimal").type(AvroUtils.wrapAsNullable(AvroUtils._decimal())).noDefault() //
.name("Test_BigDecimal").prop(TALEND6_COLUMN_TALEND_TYPE, "id_BigDecimal")
.type(AvroUtils.wrapAsNullable(AvroUtils._decimal())).noDefault() //
.name("Test_Date").prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'")
.type(AvroUtils.wrapAsNullable(AvroUtils._date())).noDefault() //
.name("valid").type().booleanType().noDefault() //
.prop(TALEND6_COLUMN_TALEND_TYPE, "id_Date").type(AvroUtils.wrapAsNullable(AvroUtils._date())).noDefault() //
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
.endRecord(); //
Schema designSchema = SchemaBuilder.builder().record("Record").prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true") //
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0") //
.prop(DiSchemaConstants.TALEND6_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'") //
.fields() //
.name("valid").type().booleanType().noDefault() //
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
.endRecord();
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
@@ -699,6 +706,36 @@ public class IncomingSchemaEnforcerTest {
assertThat(record.get(1), is((Object) new Date(1234567891011L)));
}
/**
* Checks {@link IncomingSchemaEnforcer#addDynamicField(String, String, String, String, String, String, boolean)}
* adds information about original type used in source database to runtime schema
*/
@Test
public void testAddDynamicFieldDbTypePresent() {
List<Schema.Field> fields = new ArrayList<>();
Schema.Field field1 = new Schema.Field("StringField", Schema.create(Schema.Type.STRING), null, (Object) null);
field1.addProp(TALEND6_COLUMN_TALEND_TYPE, "id_String");
field1.addProp(TALEND_COLUMN_DB_TYPE, "TEXT");
fields.add(field1);
Schema.Field field2 = new Schema.Field("IntField", Schema.create(Schema.Type.INT), null, (Object) null);
field2.addProp(TALEND6_COLUMN_TALEND_TYPE, "id_Integer");
field2.addProp(TALEND_COLUMN_DB_TYPE, "INT32");
fields.add(field2);
Schema expectedRuntimeSchema = Schema.createRecord("Record", null, null, false, fields);
Schema designSchema = SchemaBuilder.builder().record("Record").prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true")
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0").fields().endRecord();
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
enforcer.addDynamicField("StringField", "id_String", null, null, null, "TEXT", false);
enforcer.addDynamicField("IntField", "id_Integer", null, null, null, "INT32", false);
enforcer.createRuntimeSchema();
Schema actualRuntimeSchema = enforcer.getRuntimeSchema();
assertEquals(expectedRuntimeSchema, actualRuntimeSchema);
}
/**
* Checks {@link IncomingSchemaEnforcer#put()} converts string value to date according pattern specified in dynamic field
* TODO (iv.gonchar): this is incorrect behavior, because avro record should not contain java.util.Date value. It should store
@@ -710,7 +747,7 @@ public class IncomingSchemaEnforcerTest {
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0") //
.prop(DiSchemaConstants.TALEND6_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'") //
.fields() //
.name("valid").type().booleanType().noDefault() //
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
.endRecord();
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);
@@ -736,7 +773,7 @@ public class IncomingSchemaEnforcerTest {
.prop(DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0") //
.prop(DiSchemaConstants.TALEND6_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'") //
.fields() //
.name("valid").type().booleanType().noDefault() //
.name("valid").prop(TALEND6_COLUMN_TALEND_TYPE, "id_Boolean").type().booleanType().noDefault() //
.endRecord();
IncomingSchemaEnforcer enforcer = new IncomingSchemaEnforcer(designSchema);

View File

@@ -1268,7 +1268,7 @@ public class Component extends AbstractBasicComponent {
moduleNeeded = new ModuleNeeded(getName(), "", true, "mvn:org.slf4j/slf4j-log4j12/1.7.25");
componentImportNeedsList.add(moduleNeeded);
}
moduleNeeded = new ModuleNeeded(getName(), "", true, "mvn:org.talend.libraries/talend-codegen-utils/0.28.0");
moduleNeeded = new ModuleNeeded(getName(), "", true, "mvn:org.talend.libraries/talend-codegen-utils/0.28.2-SNAPSHOT");
componentImportNeedsList.add(moduleNeeded);
return componentImportNeedsList;
}

View File

@@ -6,7 +6,7 @@
context="plugin:org.talend.libraries.custom"
language="java"
message="Library for components API"
name="talend-codegen-utils.jar" mvn_uri="mvn:org.talend.libraries/talend-codegen-utils/0.28.0"
name="talend-codegen-utils.jar" mvn_uri="mvn:org.talend.libraries/talend-codegen-utils/0.28.2-SNAPSHOT"
required="true">
</libraryNeeded>
</extension>

View File

@@ -29,7 +29,7 @@
<artifactItem>
<groupId>org.talend.libraries</groupId>
<artifactId>talend-codegen-utils</artifactId>
<version>0.28.0</version>
<version>0.28.2-SNAPSHOT</version>
</artifactItem>
<artifactItem>
<groupId>org.talend.libraries</groupId>