jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-23 21:03:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

Convert source mysql decimal number with 0 decimal digits to an int (#25460)

Browse Source 
* Convert source mysql decimal number with 0 decimal point to an int

* Put int representation in json for decimal with 0 decimal digits

* bump docker image ver and release note

* auto-bump connector version

---------

Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
This commit is contained in:
Rodi Reich Zilberman
2023-05-10 16:19:57 -07:00
committed by GitHub
parent 13f2f4ad3e
commit e598cc9a6c
11 changed files with 57 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java
View File

@@ -34,6 +34,7 @@ import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME;
import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_SIZE;
import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE;
import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME;
import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_DECIMAL_DIGITS;
import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME;
import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME;
@@ -106,7 +107,13 @@ public class MySqlSourceOperations extends AbstractJdbcCompatibleSourceOperation
case BIGINT, BIGINT_UNSIGNED -> putBigInt(json, columnName, resultSet, colIndex);
case FLOAT, FLOAT_UNSIGNED -> putFloat(json, columnName, resultSet, colIndex);
case DOUBLE, DOUBLE_UNSIGNED -> putDouble(json, columnName, resultSet, colIndex);
case DECIMAL, DECIMAL_UNSIGNED -> putBigDecimal(json, columnName, resultSet, colIndex);
case DECIMAL, DECIMAL_UNSIGNED -> {
if (field.getDecimals() == 0) {
putBigInt(json, columnName, resultSet, colIndex);
} else {
putBigDecimal(json, columnName, resultSet, colIndex);
}
}
case DATE -> putDate(json, columnName, resultSet, colIndex);
case DATETIME -> putTimestamp(json, columnName, resultSet, colIndex);
case TIMESTAMP -> putTimestampWithTimezone(json, columnName, resultSet, colIndex);
@@ -181,8 +188,17 @@ public class MySqlSourceOperations extends AbstractJdbcCompatibleSourceOperation
// When CHAR[N] and VARCHAR[N] columns have binary character set, the returned
// types are BINARY[N] and VARBINARY[N], respectively. So we don't need to
// convert them here. This is verified in MySqlSourceDatatypeTest.
case DECIMAL -> {
if (field.get(INTERNAL_DECIMAL_DIGITS) != null && field.get(INTERNAL_DECIMAL_DIGITS).asInt() == 0) {
return BIGINT;
}
}
case DECIMAL_UNSIGNED -> {
if (field.get(INTERNAL_DECIMAL_DIGITS) != null && field.get(INTERNAL_DECIMAL_DIGITS).asInt() == 0) {
return BIGINT_UNSIGNED;
}
}
}
return literalType;
} catch (final IllegalArgumentException ex) {
LOGGER.warn(String.format("Could not convert column: %s from table: %s.%s with type: %s (type name: %s). Casting to VARCHAR.",

9
airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java
View File

@@ -241,6 +241,15 @@ public abstract class AbstractMySqlSourceDatatypeTest extends AbstractSourceData
.addExpectedValues("1700000.01")
.build());
addDataTypeTestData(
TestDataHolder.builder()
.sourceType("decimal")
.airbyteType(JsonSchemaType.INTEGER)
.fullSourceDataType("decimal(32,0)")
.addInsertValues("1700000.01")
.addExpectedValues("1700000")
.build());
for (final String type : Set.of("date", "date not null default '0000-00-00'")) {
addDataTypeTestData(
TestDataHolder.builder()

25
airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java
View File

@@ -81,13 +81,12 @@ public class MySqlDatatypeAccuracyTest extends AbstractMySqlSourceDatatypeTest {
for (final MysqlType mst : MysqlType.values()) {
switch (mst) {
case DECIMAL -> {
/* https://github.com/airbytehq/airbyte/issues/25027 */
// addDataTypeTestData(
// TestDataHolder.builder()
// .sourceType(mst.name())
// .airbyteType(JsonSchemaType.INTEGER)
// .fullSourceDataType("%s(1,0)".formatted(mst.getName()))
// .build());
addDataTypeTestData(
TestDataHolder.builder()
.sourceType(mst.name())
.airbyteType(JsonSchemaType.INTEGER)
.fullSourceDataType("%s(10,0)".formatted(mst.getName()))
.build());
addDataTypeTestData(
TestDataHolder.builder()
@@ -97,12 +96,12 @@ public class MySqlDatatypeAccuracyTest extends AbstractMySqlSourceDatatypeTest {
.build());
}
case DECIMAL_UNSIGNED -> {
// addDataTypeTestData(
// TestDataHolder.builder()
// .sourceType(mst.name())
// .airbyteType(JsonSchemaType.INTEGER)
// .fullSourceDataType("DECIMAL(1,0) UNSIGNED")
// .build());
addDataTypeTestData(
TestDataHolder.builder()
.sourceType(mst.name())
.airbyteType(JsonSchemaType.INTEGER)
.fullSourceDataType("DECIMAL(32,0) UNSIGNED")
.build());
addDataTypeTestData(
TestDataHolder.builder()