From daebcdb893b30f6bd7f4a90f8424723b07f01cb6 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Mon, 31 Mar 2025 09:08:42 -0700 Subject: [PATCH] Destination S3: Don't drop trailing 0 in decimals in JSONL/CSV mode (#56458) --- .../MockBasicFunctionalityIntegrationTest.kt | 1 - .../io/airbyte/cdk/load/util/JsonUtils.kt | 2 ++ .../BasicFunctionalityIntegrationTest.kt | 19 ++----------------- .../connectors/destination-s3/metadata.yaml | 2 +- .../destination/s3_v2/S3V2WriteTest.kt | 15 --------------- docs/integrations/destinations/s3.md | 1 + 6 files changed, 6 insertions(+), 34 deletions(-) diff --git a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt index 75617beec35..c1826004122 100644 --- a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt @@ -32,7 +32,6 @@ class MockBasicFunctionalityIntegrationTest : supportFileTransfer = false, commitDataIncrementally = false, allTypesBehavior = Untyped, - integralNumbersAreConvertedToInt = true, ) { @Test override fun testBasicWrite() { diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/JsonUtils.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/JsonUtils.kt index 7fea7a48bd7..da7d10975b8 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/JsonUtils.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/JsonUtils.kt @@ -10,6 +10,7 @@ import com.fasterxml.jackson.core.StreamReadConstraints import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.cfg.JsonNodeFeature import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.afterburner.AfterburnerModule import com.fasterxml.jackson.module.kotlin.registerKotlinModule @@ -28,6 +29,7 @@ object Jsons : ObjectMapper() { configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true) configure(JsonGenerator.Feature.WRITE_BIGDECIMAL_AS_PLAIN, true) + configure(JsonNodeFeature.STRIP_TRAILING_BIGDECIMAL_ZEROES, false) factory.setStreamReadConstraints( StreamReadConstraints.builder().maxStringLength(JSON_MAX_LENGTH).build() ) diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt index bace41ae155..eeb2b4d57d8 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt @@ -214,11 +214,6 @@ abstract class BasicFunctionalityIntegrationTest( val commitDataIncrementally: Boolean, val allTypesBehavior: AllTypesBehavior, val unknownTypesBehavior: UnknownTypesBehavior = UnknownTypesBehavior.PASS_THROUGH, - /** - * Some destinations implicitly convert numbers to int when it's lossless (e.g. converting 42.0 - * to 42). Those destinations should set [integralNumbersAreConvertedToInt] to `true`. - */ - val integralNumbersAreConvertedToInt: Boolean = false, nullEqualsUnset: Boolean = false, configUpdater: ConfigurationUpdater = FakeConfigurationUpdater, ) : @@ -2243,12 +2238,7 @@ abstract class BasicFunctionalityIntegrationTest( mapOf( "id" to 8, "integer" to 0, - "number" to - if (integralNumbersAreConvertedToInt) { - 0 - } else { - 0.0 - }, + "number" to 0.0, ), airbyteMeta = OutputRecord.Meta(syncId = 42), ), @@ -2259,12 +2249,7 @@ abstract class BasicFunctionalityIntegrationTest( mapOf( "id" to 9, "integer" to -1, - "number" to - if (integralNumbersAreConvertedToInt) { - -1 - } else { - -1.0 - }, + "number" to -1.0, ), airbyteMeta = OutputRecord.Meta(syncId = 42), ), diff --git a/airbyte-integrations/connectors/destination-s3/metadata.yaml b/airbyte-integrations/connectors/destination-s3/metadata.yaml index d316660e947..8aa253ea4ca 100644 --- a/airbyte-integrations/connectors/destination-s3/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 - dockerImageTag: 1.5.8 + dockerImageTag: 1.6.0 dockerRepository: airbyte/destination-s3 githubIssueLabel: destination-s3 icon: s3.svg diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt b/airbyte-integrations/connectors/destination-s3/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt index ee9f8e4aa96..b87777d681e 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt @@ -43,7 +43,6 @@ abstract class S3V2WriteTest( nullEqualsUnset: Boolean = false, unknownTypesBehavior: UnknownTypesBehavior = UnknownTypesBehavior.PASS_THROUGH, private val mergesUnions: Boolean = false, - integralNumbersAreConvertedToInt: Boolean, ) : BasicFunctionalityIntegrationTest( S3V2TestUtils.getConfig(path), @@ -65,7 +64,6 @@ abstract class S3V2WriteTest( nullEqualsUnset = nullEqualsUnset, supportFileTransfer = true, unknownTypesBehavior = unknownTypesBehavior, - integralNumbersAreConvertedToInt = integralNumbersAreConvertedToInt, ) { @Disabled("Irrelevant for file destinations") @Test @@ -317,7 +315,6 @@ class S3V2WriteTestJsonUncompressed : schematizedArrayBehavior = SchematizedNestedValueBehavior.PASS_THROUGH, preserveUndeclaredFields = true, allTypesBehavior = Untyped, - integralNumbersAreConvertedToInt = true, ) class S3V2WriteTestJsonRootLevelFlattening : @@ -330,7 +327,6 @@ class S3V2WriteTestJsonRootLevelFlattening : schematizedArrayBehavior = SchematizedNestedValueBehavior.PASS_THROUGH, preserveUndeclaredFields = true, allTypesBehavior = Untyped, - integralNumbersAreConvertedToInt = true, ) class S3V2WriteTestJsonGzip : @@ -343,7 +339,6 @@ class S3V2WriteTestJsonGzip : schematizedArrayBehavior = SchematizedNestedValueBehavior.PASS_THROUGH, preserveUndeclaredFields = true, allTypesBehavior = Untyped, - integralNumbersAreConvertedToInt = true, ) class S3V2WriteTestCsvUncompressed : @@ -356,7 +351,6 @@ class S3V2WriteTestCsvUncompressed : schematizedArrayBehavior = SchematizedNestedValueBehavior.PASS_THROUGH, preserveUndeclaredFields = true, allTypesBehavior = Untyped, - integralNumbersAreConvertedToInt = true, ) { @Test override fun testBasicWriteFile() { @@ -376,7 +370,6 @@ class S3V2WriteTestCsvRootLevelFlattening : allTypesBehavior = Untyped, nullEqualsUnset = true, // Technically true of unflattened as well, but no top-level fields are nullable - integralNumbersAreConvertedToInt = true, ) class S3V2WriteTestCsvGzip : @@ -389,7 +382,6 @@ class S3V2WriteTestCsvGzip : schematizedArrayBehavior = SchematizedNestedValueBehavior.PASS_THROUGH, preserveUndeclaredFields = true, allTypesBehavior = Untyped, - integralNumbersAreConvertedToInt = true, ) class S3V2WriteTestAvroUncompressed : @@ -405,7 +397,6 @@ class S3V2WriteTestAvroUncompressed : nullEqualsUnset = true, unknownTypesBehavior = UnknownTypesBehavior.FAIL, mergesUnions = true, - integralNumbersAreConvertedToInt = false, ) class S3V2WriteTestAvroBzip2 : @@ -421,7 +412,6 @@ class S3V2WriteTestAvroBzip2 : nullEqualsUnset = true, unknownTypesBehavior = UnknownTypesBehavior.FAIL, mergesUnions = true, - integralNumbersAreConvertedToInt = false, ) class S3V2WriteTestParquetUncompressed : @@ -437,7 +427,6 @@ class S3V2WriteTestParquetUncompressed : nullEqualsUnset = true, unknownTypesBehavior = UnknownTypesBehavior.FAIL, mergesUnions = true, - integralNumbersAreConvertedToInt = false, ) class S3V2WriteTestParquetSnappy : @@ -453,7 +442,6 @@ class S3V2WriteTestParquetSnappy : nullEqualsUnset = true, unknownTypesBehavior = UnknownTypesBehavior.FAIL, mergesUnions = true, - integralNumbersAreConvertedToInt = false, ) class S3V2WriteTestEndpointURL : @@ -468,7 +456,6 @@ class S3V2WriteTestEndpointURL : preserveUndeclaredFields = false, allTypesBehavior = Untyped, nullEqualsUnset = true, - integralNumbersAreConvertedToInt = true, ) class S3V2AmbiguousFilepath : @@ -482,7 +469,6 @@ class S3V2AmbiguousFilepath : schematizedArrayBehavior = SchematizedNestedValueBehavior.PASS_THROUGH, preserveUndeclaredFields = true, allTypesBehavior = Untyped, - integralNumbersAreConvertedToInt = true, ) class S3V2CsvAssumeRole : @@ -495,5 +481,4 @@ class S3V2CsvAssumeRole : schematizedArrayBehavior = SchematizedNestedValueBehavior.PASS_THROUGH, preserveUndeclaredFields = true, allTypesBehavior = Untyped, - integralNumbersAreConvertedToInt = true, ) diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index fbd5dc2bd04..c3001aa6996 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -544,6 +544,7 @@ To see connector limitations, or troubleshoot your S3 connector, see more [in ou | Version | Date | Pull Request | Subject | |:------------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.6.0 | 2025-03-28 | [56458](https://github.com/airbytehq/airbyte/pull/56458) | Do not drop trailing `.0` from decimals in CSV/JSONL data | | 1.5.8 | 2025-03-25 | [56398](https://github.com/airbytehq/airbyte/pull/56398) | Internal CDK change to mark old Avro and Parquet formats as deprecated | | 1.5.7 | 2025-03-24 | [56355](https://github.com/airbytehq/airbyte/pull/56355) | Upgrade to airbyte/java-connector-base:2.0.1 to be M4 compatible. | | 1.5.6 | 2025-03-24 | [55849](https://github.com/airbytehq/airbyte/pull/55849) | Internal refactoring |