diff --git a/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/data/AirbyteType.kt b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/data/AirbyteSchemaType.kt similarity index 90% rename from airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/data/AirbyteType.kt rename to airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/data/AirbyteSchemaType.kt index e74a74d07bf..ed7c9426529 100644 --- a/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/data/AirbyteType.kt +++ b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/data/AirbyteSchemaType.kt @@ -12,7 +12,7 @@ import io.airbyte.protocol.models.JsonSchemaType * This maps to the subset of [JsonSchemaType] which is used in practice. Its main reason for * existing is to provide type-safety and convenient comparisons and string representations. */ -sealed interface AirbyteType { +sealed interface AirbyteSchemaType { /** Unwraps the underlying Airbyte protocol type object. */ fun asJsonSchemaType(): JsonSchemaType @@ -20,18 +20,18 @@ sealed interface AirbyteType { fun asJsonSchema(): JsonNode = Jsons.valueToTree(asJsonSchemaType().jsonSchemaTypeMap) } -data class ArrayAirbyteType( - val item: AirbyteType, -) : AirbyteType { +data class ArrayAirbyteSchemaType( + val item: AirbyteSchemaType, +) : AirbyteSchemaType { override fun asJsonSchemaType(): JsonSchemaType = JsonSchemaType.builder(JsonSchemaPrimitiveUtil.JsonSchemaPrimitive.ARRAY) .withItems(item.asJsonSchemaType()) .build() } -enum class LeafAirbyteType( +enum class LeafAirbyteSchemaType( private val jsonSchemaType: JsonSchemaType, -) : AirbyteType { +) : AirbyteSchemaType { BOOLEAN(JsonSchemaType.BOOLEAN), STRING(JsonSchemaType.STRING), BINARY(JsonSchemaType.STRING_BASE_64), diff --git a/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/discover/AirbyteStreamFactory.kt b/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/discover/AirbyteStreamFactory.kt index 4dd2b568225..b238524f4e9 100644 --- a/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/discover/AirbyteStreamFactory.kt +++ b/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/discover/AirbyteStreamFactory.kt @@ -20,7 +20,7 @@ interface AirbyteStreamFactory { discoveredStream.id.name, discoveredStream.id.namespace, discoveredStream.columns.map { - AirbyteField.of(it.id, it.type.airbyteType.asJsonSchemaType()) + AirbyteField.of(it.id, it.type.airbyteSchemaType.asJsonSchemaType()) }, ) } diff --git a/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/discover/Field.kt b/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/discover/Field.kt index 364c32b43ba..5eec0d556a5 100644 --- a/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/discover/Field.kt +++ b/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/discover/Field.kt @@ -1,12 +1,12 @@ /* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ package io.airbyte.cdk.discover -import io.airbyte.cdk.data.AirbyteType +import io.airbyte.cdk.data.AirbyteSchemaType import io.airbyte.cdk.data.IntCodec import io.airbyte.cdk.data.JsonDecoder import io.airbyte.cdk.data.JsonEncoder import io.airbyte.cdk.data.JsonStringCodec -import io.airbyte.cdk.data.LeafAirbyteType +import io.airbyte.cdk.data.LeafAirbyteSchemaType import io.airbyte.cdk.data.OffsetDateTimeCodec import java.time.OffsetDateTime @@ -23,7 +23,7 @@ sealed interface FieldOrMetaField { */ interface FieldType { /** maps to [io.airbyte.protocol.models.Field.type] */ - val airbyteType: AirbyteType + val airbyteSchemaType: AirbyteSchemaType val jsonEncoder: JsonEncoder<*> } @@ -73,19 +73,20 @@ enum class CommonMetaField( } data object CdcStringMetaFieldType : LosslessFieldType { - override val airbyteType: AirbyteType = LeafAirbyteType.STRING + override val airbyteSchemaType: AirbyteSchemaType = LeafAirbyteSchemaType.STRING override val jsonEncoder: JsonEncoder = JsonStringCodec override val jsonDecoder: JsonDecoder = JsonStringCodec } data object CdcIntegerMetaFieldType : LosslessFieldType { - override val airbyteType: AirbyteType = LeafAirbyteType.INTEGER + override val airbyteSchemaType: AirbyteSchemaType = LeafAirbyteSchemaType.INTEGER override val jsonEncoder: JsonEncoder = IntCodec override val jsonDecoder: JsonDecoder = IntCodec } data object CdcOffsetDateTimeMetaFieldType : LosslessFieldType { - override val airbyteType: AirbyteType = LeafAirbyteType.TIMESTAMP_WITH_TIMEZONE + override val airbyteSchemaType: AirbyteSchemaType = + LeafAirbyteSchemaType.TIMESTAMP_WITH_TIMEZONE override val jsonEncoder: JsonEncoder = OffsetDateTimeCodec override val jsonDecoder: JsonDecoder = OffsetDateTimeCodec } diff --git a/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/output/CatalogValidationFailureHandler.kt b/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/output/CatalogValidationFailureHandler.kt index 5771b07934d..61e30795488 100644 --- a/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/output/CatalogValidationFailureHandler.kt +++ b/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/output/CatalogValidationFailureHandler.kt @@ -2,7 +2,7 @@ package io.airbyte.cdk.output import io.airbyte.cdk.StreamIdentifier -import io.airbyte.cdk.data.AirbyteType +import io.airbyte.cdk.data.AirbyteSchemaType import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.DefaultImplementation import jakarta.inject.Singleton @@ -43,8 +43,8 @@ data class FieldNotFound( data class FieldTypeMismatch( override val streamID: StreamIdentifier, val fieldName: String, - val expected: AirbyteType, - val actual: AirbyteType, + val expected: AirbyteSchemaType, + val actual: AirbyteSchemaType, ) : CatalogValidationFailure data class InvalidPrimaryKey( diff --git a/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/read/StateManagerFactory.kt b/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/read/StateManagerFactory.kt index c142c92d307..95140bcc972 100644 --- a/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/read/StateManagerFactory.kt +++ b/airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/read/StateManagerFactory.kt @@ -10,9 +10,9 @@ import io.airbyte.cdk.command.GlobalInputState import io.airbyte.cdk.command.InputState import io.airbyte.cdk.command.SourceConfiguration import io.airbyte.cdk.command.StreamInputState -import io.airbyte.cdk.data.AirbyteType -import io.airbyte.cdk.data.ArrayAirbyteType -import io.airbyte.cdk.data.LeafAirbyteType +import io.airbyte.cdk.data.AirbyteSchemaType +import io.airbyte.cdk.data.ArrayAirbyteSchemaType +import io.airbyte.cdk.data.LeafAirbyteSchemaType import io.airbyte.cdk.discover.CommonMetaField import io.airbyte.cdk.discover.Field import io.airbyte.cdk.discover.FieldOrMetaField @@ -132,7 +132,7 @@ class StateManagerFactory( } } - val expectedSchema: Map = + val expectedSchema: Map = jsonSchemaProperties.properties().associate { (id: String, schema: JsonNode) -> id to airbyteTypeFromJsonSchema(schema) } @@ -150,15 +150,15 @@ class StateManagerFactory( handler.accept(FieldNotFound(streamID, id)) return null } - val expectedAirbyteType: AirbyteType = expectedSchema[id] ?: return null - val actualAirbyteType: AirbyteType = actualColumn.type.airbyteType - if (expectedAirbyteType != actualAirbyteType) { + val expectedAirbyteSchemaType: AirbyteSchemaType = expectedSchema[id] ?: return null + val actualAirbyteSchemaType: AirbyteSchemaType = actualColumn.type.airbyteSchemaType + if (expectedAirbyteSchemaType != actualAirbyteSchemaType) { handler.accept( FieldTypeMismatch( streamID, id, - expectedAirbyteType, - actualAirbyteType, + expectedAirbyteSchemaType, + actualAirbyteSchemaType, ), ) return null @@ -229,44 +229,44 @@ class StateManagerFactory( } /** - * Recursively re-generates the original [AirbyteType] from a catalog stream field's JSON + * Recursively re-generates the original [AirbyteSchemaType] from a catalog stream field's JSON * schema. */ - private fun airbyteTypeFromJsonSchema(jsonSchema: JsonNode): AirbyteType { + private fun airbyteTypeFromJsonSchema(jsonSchema: JsonNode): AirbyteSchemaType { fun value(key: String): String = jsonSchema[key]?.asText() ?: "" return when (value("type")) { - "array" -> ArrayAirbyteType(airbyteTypeFromJsonSchema(jsonSchema["items"])) - "null" -> LeafAirbyteType.NULL - "boolean" -> LeafAirbyteType.BOOLEAN + "array" -> ArrayAirbyteSchemaType(airbyteTypeFromJsonSchema(jsonSchema["items"])) + "null" -> LeafAirbyteSchemaType.NULL + "boolean" -> LeafAirbyteSchemaType.BOOLEAN "number" -> when (value("airbyte_type")) { "integer", - "big_integer", -> LeafAirbyteType.INTEGER - else -> LeafAirbyteType.NUMBER + "big_integer", -> LeafAirbyteSchemaType.INTEGER + else -> LeafAirbyteSchemaType.NUMBER } "string" -> when (value("format")) { - "date" -> LeafAirbyteType.DATE + "date" -> LeafAirbyteSchemaType.DATE "date-time" -> if (value("airbyte_type") == "timestamp_with_timezone") { - LeafAirbyteType.TIMESTAMP_WITH_TIMEZONE + LeafAirbyteSchemaType.TIMESTAMP_WITH_TIMEZONE } else { - LeafAirbyteType.TIMESTAMP_WITHOUT_TIMEZONE + LeafAirbyteSchemaType.TIMESTAMP_WITHOUT_TIMEZONE } "time" -> if (value("airbyte_type") == "time_with_timezone") { - LeafAirbyteType.TIME_WITH_TIMEZONE + LeafAirbyteSchemaType.TIME_WITH_TIMEZONE } else { - LeafAirbyteType.TIME_WITHOUT_TIMEZONE + LeafAirbyteSchemaType.TIME_WITHOUT_TIMEZONE } else -> if (value("contentEncoding") == "base64") { - LeafAirbyteType.BINARY + LeafAirbyteSchemaType.BINARY } else { - LeafAirbyteType.STRING + LeafAirbyteSchemaType.STRING } } - else -> LeafAirbyteType.JSONB + else -> LeafAirbyteSchemaType.JSONB } } } diff --git a/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/IntFieldType.kt b/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/IntFieldType.kt index 821082136bd..c1bf7c442f9 100644 --- a/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/IntFieldType.kt +++ b/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/IntFieldType.kt @@ -4,12 +4,12 @@ package io.airbyte.cdk.discover -import io.airbyte.cdk.data.AirbyteType +import io.airbyte.cdk.data.AirbyteSchemaType import io.airbyte.cdk.data.IntCodec import io.airbyte.cdk.data.JsonEncoder -import io.airbyte.cdk.data.LeafAirbyteType +import io.airbyte.cdk.data.LeafAirbyteSchemaType data object IntFieldType : FieldType { - override val airbyteType: AirbyteType = LeafAirbyteType.INTEGER + override val airbyteSchemaType: AirbyteSchemaType = LeafAirbyteSchemaType.INTEGER override val jsonEncoder: JsonEncoder<*> = IntCodec } diff --git a/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/OffsetDateTimeFieldType.kt b/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/OffsetDateTimeFieldType.kt index ecf3c203539..fc25e61939f 100644 --- a/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/OffsetDateTimeFieldType.kt +++ b/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/OffsetDateTimeFieldType.kt @@ -4,12 +4,13 @@ package io.airbyte.cdk.discover -import io.airbyte.cdk.data.AirbyteType +import io.airbyte.cdk.data.AirbyteSchemaType import io.airbyte.cdk.data.JsonEncoder -import io.airbyte.cdk.data.LeafAirbyteType +import io.airbyte.cdk.data.LeafAirbyteSchemaType import io.airbyte.cdk.data.OffsetDateTimeCodec data object OffsetDateTimeFieldType : FieldType { - override val airbyteType: AirbyteType = LeafAirbyteType.TIMESTAMP_WITH_TIMEZONE + override val airbyteSchemaType: AirbyteSchemaType = + LeafAirbyteSchemaType.TIMESTAMP_WITH_TIMEZONE override val jsonEncoder: JsonEncoder<*> = OffsetDateTimeCodec } diff --git a/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/StringFieldType.kt b/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/StringFieldType.kt index 8f1d386cf2e..e5097c873e2 100644 --- a/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/StringFieldType.kt +++ b/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/StringFieldType.kt @@ -4,12 +4,12 @@ package io.airbyte.cdk.discover -import io.airbyte.cdk.data.AirbyteType +import io.airbyte.cdk.data.AirbyteSchemaType import io.airbyte.cdk.data.JsonEncoder -import io.airbyte.cdk.data.LeafAirbyteType +import io.airbyte.cdk.data.LeafAirbyteSchemaType import io.airbyte.cdk.data.TextCodec data object StringFieldType : FieldType { - override val airbyteType: AirbyteType = LeafAirbyteType.STRING + override val airbyteSchemaType: AirbyteSchemaType = LeafAirbyteSchemaType.STRING override val jsonEncoder: JsonEncoder<*> = TextCodec } diff --git a/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/TestAirbyteStreamFactory.kt b/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/TestAirbyteStreamFactory.kt index 5b84e88c2fa..c7d9c566460 100644 --- a/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/TestAirbyteStreamFactory.kt +++ b/airbyte-cdk/bulk/core/extract/src/testFixtures/kotlin/io/airbyte/cdk/discover/TestAirbyteStreamFactory.kt @@ -21,7 +21,7 @@ class TestAirbyteStreamFactory : AirbyteStreamFactory { supportedSyncModes = listOf(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL) (jsonSchema["properties"] as ObjectNode).apply { for (metaField in CommonMetaField.entries) { - set(metaField.id, metaField.type.airbyteType.asJsonSchema()) + set(metaField.id, metaField.type.airbyteSchemaType.asJsonSchema()) } } defaultCursorField = listOf(CommonMetaField.CDC_LSN.id) diff --git a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt similarity index 53% rename from airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt rename to airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt index 0513c31bdf0..ba8f1a8e315 100644 --- a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt @@ -2,12 +2,12 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.mock_integration_test +package io.airbyte.cdk.load.mock_integration_test -import io.airbyte.cdk.test.util.NoopDestinationCleaner -import io.airbyte.cdk.test.util.NoopExpectedRecordMapper -import io.airbyte.cdk.test.util.NoopNameMapper -import io.airbyte.cdk.test.write.BasicFunctionalityIntegrationTest +import io.airbyte.cdk.load.test.util.NoopDestinationCleaner +import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper +import io.airbyte.cdk.load.test.util.NoopNameMapper +import io.airbyte.cdk.load.write.BasicFunctionalityIntegrationTest class MockBasicFunctionalityIntegrationTest : BasicFunctionalityIntegrationTest( diff --git a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockDestinationBackend.kt b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationBackend.kt similarity index 84% rename from airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockDestinationBackend.kt rename to airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationBackend.kt index 99353873200..fb7720bac2f 100644 --- a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockDestinationBackend.kt +++ b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationBackend.kt @@ -2,10 +2,10 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.mock_integration_test +package io.airbyte.cdk.load.mock_integration_test -import io.airbyte.cdk.test.util.DestinationDataDumper -import io.airbyte.cdk.test.util.OutputRecord +import io.airbyte.cdk.load.test.util.DestinationDataDumper +import io.airbyte.cdk.load.test.util.OutputRecord import java.util.concurrent.ConcurrentHashMap object MockDestinationBackend { diff --git a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockDestinationChecker.kt b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationChecker.kt similarity index 71% rename from airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockDestinationChecker.kt rename to airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationChecker.kt index 824e880dc13..9972357704d 100644 --- a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockDestinationChecker.kt +++ b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationChecker.kt @@ -2,9 +2,9 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.mock_integration_test +package io.airbyte.cdk.load.mock_integration_test -import io.airbyte.cdk.check.DestinationChecker +import io.airbyte.cdk.load.check.DestinationChecker import javax.inject.Singleton @Singleton diff --git a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockDestinationConfiguration.kt b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationConfiguration.kt similarity index 83% rename from airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockDestinationConfiguration.kt rename to airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationConfiguration.kt index ac92cf54aba..c582e5e463d 100644 --- a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockDestinationConfiguration.kt +++ b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationConfiguration.kt @@ -2,11 +2,11 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.mock_integration_test +package io.airbyte.cdk.load.mock_integration_test import io.airbyte.cdk.command.ConfigurationSpecification -import io.airbyte.cdk.command.DestinationConfiguration -import io.airbyte.cdk.command.DestinationConfigurationFactory +import io.airbyte.cdk.load.command.DestinationConfiguration +import io.airbyte.cdk.load.command.DestinationConfigurationFactory import io.micronaut.context.annotation.Factory import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockDestinationWriter.kt b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationWriter.kt similarity index 82% rename from airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockDestinationWriter.kt rename to airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationWriter.kt index 7b8c5c4522f..c49115cb2c1 100644 --- a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/mock_integration_test/MockDestinationWriter.kt +++ b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationWriter.kt @@ -2,16 +2,16 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.mock_integration_test +package io.airbyte.cdk.load.mock_integration_test -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.data.ObjectValue -import io.airbyte.cdk.message.Batch -import io.airbyte.cdk.message.DestinationRecord -import io.airbyte.cdk.message.SimpleBatch -import io.airbyte.cdk.test.util.OutputRecord -import io.airbyte.cdk.write.DestinationWriter -import io.airbyte.cdk.write.StreamLoader +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.data.ObjectValue +import io.airbyte.cdk.load.message.Batch +import io.airbyte.cdk.load.message.DestinationRecord +import io.airbyte.cdk.load.message.SimpleBatch +import io.airbyte.cdk.load.test.util.OutputRecord +import io.airbyte.cdk.load.write.DestinationWriter +import io.airbyte.cdk.load.write.StreamLoader import java.time.Instant import java.util.UUID import javax.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/check/CheckOperation.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/check/CheckOperation.kt similarity index 92% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/check/CheckOperation.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/check/CheckOperation.kt index eae3f262a78..86b1c6a1bef 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/check/CheckOperation.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/check/CheckOperation.kt @@ -2,13 +2,13 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.check +package io.airbyte.cdk.load.check import io.airbyte.cdk.Operation import io.airbyte.cdk.command.ConfigurationSpecification import io.airbyte.cdk.command.ConfigurationSpecificationSupplier -import io.airbyte.cdk.command.DestinationConfiguration -import io.airbyte.cdk.command.DestinationConfigurationFactory +import io.airbyte.cdk.load.command.DestinationConfiguration +import io.airbyte.cdk.load.command.DestinationConfigurationFactory import io.airbyte.cdk.output.ExceptionHandler import io.airbyte.cdk.output.OutputConsumer import io.airbyte.protocol.models.v0.AirbyteConnectionStatus diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/check/DestinationChecker.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/check/DestinationChecker.kt similarity index 89% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/check/DestinationChecker.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/check/DestinationChecker.kt index d3dffe06015..bbe65d621e8 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/check/DestinationChecker.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/check/DestinationChecker.kt @@ -2,9 +2,9 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.check +package io.airbyte.cdk.load.check -import io.airbyte.cdk.command.DestinationConfiguration +import io.airbyte.cdk.load.command.DestinationConfiguration /** * A check operation that is run before the destination is used. diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/command/DestinationCatalog.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationCatalog.kt similarity index 97% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/command/DestinationCatalog.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationCatalog.kt index 9bf46fd67c0..b771893960a 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/command/DestinationCatalog.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationCatalog.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.command +package io.airbyte.cdk.load.command import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog import io.micronaut.context.annotation.Factory diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/command/DestinationConfiguration.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationConfiguration.kt similarity index 90% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/command/DestinationConfiguration.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationConfiguration.kt index d7cc1a4f387..8efb5166648 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/command/DestinationConfiguration.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationConfiguration.kt @@ -2,8 +2,11 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.command +package io.airbyte.cdk.load.command +import io.airbyte.cdk.command.Configuration +import io.airbyte.cdk.command.ConfigurationSpecification +import io.airbyte.cdk.command.ConfigurationSpecificationSupplier import io.micronaut.context.annotation.Factory import jakarta.inject.Singleton import java.nio.file.Path diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/command/DestinationConfigurationFactory.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationConfigurationFactory.kt similarity index 87% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/command/DestinationConfigurationFactory.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationConfigurationFactory.kt index 937635acfdb..3dbf9e5999e 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/command/DestinationConfigurationFactory.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationConfigurationFactory.kt @@ -2,9 +2,10 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.command +package io.airbyte.cdk.load.command import io.airbyte.cdk.ConfigErrorException +import io.airbyte.cdk.command.ConfigurationSpecification interface DestinationConfigurationFactory< I : ConfigurationSpecification, O : DestinationConfiguration> { diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/command/DestinationStream.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationStream.kt similarity index 95% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/command/DestinationStream.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationStream.kt index cf48050cc4f..83fedeb2d95 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/command/DestinationStream.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationStream.kt @@ -2,11 +2,11 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.command +package io.airbyte.cdk.load.command -import io.airbyte.cdk.data.AirbyteType -import io.airbyte.cdk.data.AirbyteTypeToJsonSchema -import io.airbyte.cdk.data.JsonSchemaToAirbyteType +import io.airbyte.cdk.load.data.AirbyteType +import io.airbyte.cdk.load.data.AirbyteTypeToJsonSchema +import io.airbyte.cdk.load.data.JsonSchemaToAirbyteType import io.airbyte.protocol.models.v0.AirbyteStream import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream import io.airbyte.protocol.models.v0.DestinationSyncMode diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/AirbyteType.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteType.kt similarity index 96% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/AirbyteType.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteType.kt index 8824d1699f0..7d0ff0ec224 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/AirbyteType.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteType.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.data +package io.airbyte.cdk.load.data sealed interface AirbyteType diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/AirbyteTypeToJsonSchema.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteTypeToJsonSchema.kt similarity index 98% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/AirbyteTypeToJsonSchema.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteTypeToJsonSchema.kt index 7fa89feba44..93f2c5b2aa0 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/AirbyteTypeToJsonSchema.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteTypeToJsonSchema.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.data +package io.airbyte.cdk.load.data import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.JsonNodeFactory diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/AirbyteValue.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteValue.kt similarity index 99% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/AirbyteValue.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteValue.kt index 4f6040bc599..fe8215ddb09 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/AirbyteValue.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteValue.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.data +package io.airbyte.cdk.load.data import java.math.BigDecimal import java.time.LocalDate diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/AirbyteValueToJson.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteValueToJson.kt similarity index 97% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/AirbyteValueToJson.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteValueToJson.kt index 2d79157c54a..d4e688dbda6 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/AirbyteValueToJson.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteValueToJson.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.data +package io.airbyte.cdk.load.data import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.JsonNodeFactory diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/JsonSchemaToAirbyteType.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/JsonSchemaToAirbyteType.kt similarity index 99% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/JsonSchemaToAirbyteType.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/JsonSchemaToAirbyteType.kt index 14ae1967e1d..b6c67a6ee8a 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/JsonSchemaToAirbyteType.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/JsonSchemaToAirbyteType.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.data +package io.airbyte.cdk.load.data import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.JsonNodeFactory diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/JsonToAirbyteValue.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/JsonToAirbyteValue.kt similarity index 99% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/JsonToAirbyteValue.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/JsonToAirbyteValue.kt index bcce2ea1e1c..a0872b4e493 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/data/JsonToAirbyteValue.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/JsonToAirbyteValue.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.data +package io.airbyte.cdk.load.data import com.fasterxml.jackson.databind.JsonNode import java.math.BigDecimal diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/file/LocalFile.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/LocalFile.kt similarity index 97% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/file/LocalFile.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/LocalFile.kt index 67ced63db70..0701e350b49 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/file/LocalFile.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/LocalFile.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.file +package io.airbyte.cdk.load.file import io.micronaut.context.annotation.DefaultImplementation import java.io.Closeable diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/file/TempFileProvider.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/TempFileProvider.kt similarity index 94% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/file/TempFileProvider.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/TempFileProvider.kt index 0455ec323db..cdb0f353cf3 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/file/TempFileProvider.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/TempFileProvider.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.file +package io.airbyte.cdk.load.file import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/file/TimeProvider.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/TimeProvider.kt similarity index 93% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/file/TimeProvider.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/TimeProvider.kt index 9090007b83b..054bd6074b2 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/file/TimeProvider.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/TimeProvider.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.file +package io.airbyte.cdk.load.file import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/Batch.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/Batch.kt similarity index 97% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/Batch.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/Batch.kt index a99b2ad5c3d..41e4f38ce73 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/Batch.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/Batch.kt @@ -2,12 +2,12 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.message +package io.airbyte.cdk.load.message import com.google.common.collect.Range import com.google.common.collect.RangeSet import com.google.common.collect.TreeRangeSet -import io.airbyte.cdk.file.LocalFile +import io.airbyte.cdk.load.file.LocalFile /** * Represents an accumulated batch of records in some stage of processing. diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/DestinationMessage.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessage.kt similarity index 96% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/DestinationMessage.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessage.kt index 28af91cdd4b..2fd484bdb0a 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/DestinationMessage.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessage.kt @@ -2,17 +2,17 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.message +package io.airbyte.cdk.load.message import com.fasterxml.jackson.databind.JsonNode -import io.airbyte.cdk.command.DestinationCatalog -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.data.AirbyteValue -import io.airbyte.cdk.data.AirbyteValueToJson -import io.airbyte.cdk.data.JsonToAirbyteValue -import io.airbyte.cdk.data.ObjectTypeWithoutSchema -import io.airbyte.cdk.message.CheckpointMessage.Checkpoint -import io.airbyte.cdk.message.CheckpointMessage.Stats +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.data.AirbyteValue +import io.airbyte.cdk.load.data.AirbyteValueToJson +import io.airbyte.cdk.load.data.JsonToAirbyteValue +import io.airbyte.cdk.load.data.ObjectTypeWithoutSchema +import io.airbyte.cdk.load.message.CheckpointMessage.Checkpoint +import io.airbyte.cdk.load.message.CheckpointMessage.Stats import io.airbyte.protocol.models.Jsons import io.airbyte.protocol.models.v0.AirbyteGlobalState import io.airbyte.protocol.models.v0.AirbyteMessage diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/DestinationMessageDeserializer.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessageDeserializer.kt similarity index 96% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/DestinationMessageDeserializer.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessageDeserializer.kt index d052dd388cc..81a7f5a4fd8 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/DestinationMessageDeserializer.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessageDeserializer.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.message +package io.airbyte.cdk.load.message import io.airbyte.cdk.util.Jsons import io.airbyte.protocol.models.v0.AirbyteMessage diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/DestinationMessageQueues.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessageQueues.kt similarity index 91% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/DestinationMessageQueues.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessageQueues.kt index 8043d199c70..aa9099044b2 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/DestinationMessageQueues.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessageQueues.kt @@ -2,12 +2,12 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.message +package io.airbyte.cdk.load.message -import io.airbyte.cdk.command.DestinationCatalog -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.state.MemoryManager -import io.airbyte.cdk.state.Reserved +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.state.MemoryManager +import io.airbyte.cdk.load.state.Reserved import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton import java.util.concurrent.ConcurrentHashMap diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/MessageQueue.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/MessageQueue.kt similarity index 91% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/MessageQueue.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/MessageQueue.kt index 7f15270aa86..2fe9488d03a 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/message/MessageQueue.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/MessageQueue.kt @@ -2,9 +2,9 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.message +package io.airbyte.cdk.load.message -import io.airbyte.cdk.util.CloseableCoroutine +import io.airbyte.cdk.load.util.CloseableCoroutine import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.receiveAsFlow diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/spec/DestinationSpecificationInternal.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/spec/DestinationSpecificationInternal.kt similarity index 86% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/spec/DestinationSpecificationInternal.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/spec/DestinationSpecificationInternal.kt index 944a05a131f..780b4b51681 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/spec/DestinationSpecificationInternal.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/spec/DestinationSpecificationInternal.kt @@ -2,8 +2,10 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.spec +package io.airbyte.cdk.load.spec +import io.airbyte.cdk.spec.IdentitySpecificationExtender +import io.airbyte.cdk.spec.SpecificationExtender import io.airbyte.protocol.models.v0.ConnectorSpecification import io.airbyte.protocol.models.v0.DestinationSyncMode import io.micronaut.context.annotation.Replaces diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/CheckpointManager.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/CheckpointManager.kt similarity index 97% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/CheckpointManager.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/CheckpointManager.kt index 1b4302ae283..a945b27271a 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/CheckpointManager.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/CheckpointManager.kt @@ -2,14 +2,14 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.state +package io.airbyte.cdk.load.state import edu.umd.cs.findbugs.annotations.SuppressFBWarnings -import io.airbyte.cdk.command.DestinationCatalog -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.file.TimeProvider -import io.airbyte.cdk.message.CheckpointMessage -import io.airbyte.cdk.util.use +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.file.TimeProvider +import io.airbyte.cdk.load.message.CheckpointMessage +import io.airbyte.cdk.load.util.use import io.airbyte.protocol.models.v0.AirbyteMessage import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/FlushStrategy.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/FlushStrategy.kt similarity index 87% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/FlushStrategy.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/FlushStrategy.kt index cc8949aa619..a6177ae8d88 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/FlushStrategy.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/FlushStrategy.kt @@ -2,14 +2,14 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.state +package io.airbyte.cdk.load.state import com.google.common.collect.Range import edu.umd.cs.findbugs.annotations.SuppressFBWarnings -import io.airbyte.cdk.command.DestinationConfiguration -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.message.QueueReader -import io.airbyte.cdk.task.internal.ForceFlushEvent +import io.airbyte.cdk.load.command.DestinationConfiguration +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.message.QueueReader +import io.airbyte.cdk.load.task.internal.ForceFlushEvent import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton import java.util.concurrent.ConcurrentHashMap diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/MemoryManager.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/MemoryManager.kt similarity index 97% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/MemoryManager.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/MemoryManager.kt index f3c40acf143..99ed41d73ce 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/MemoryManager.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/MemoryManager.kt @@ -2,9 +2,9 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.state +package io.airbyte.cdk.load.state -import io.airbyte.cdk.util.CloseableCoroutine +import io.airbyte.cdk.load.util.CloseableCoroutine import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton import java.util.concurrent.atomic.AtomicBoolean diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/StreamManager.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/StreamManager.kt similarity index 97% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/StreamManager.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/StreamManager.kt index ee5546a5704..3931a60ccdf 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/StreamManager.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/StreamManager.kt @@ -2,14 +2,14 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.state +package io.airbyte.cdk.load.state import com.google.common.collect.Range import com.google.common.collect.RangeSet import com.google.common.collect.TreeRangeSet -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.message.Batch -import io.airbyte.cdk.message.BatchEnvelope +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.message.Batch +import io.airbyte.cdk.load.message.BatchEnvelope import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/SyncManager.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/SyncManager.kt similarity index 95% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/SyncManager.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/SyncManager.kt index 431ed0c9983..2f972f5a613 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/state/SyncManager.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/state/SyncManager.kt @@ -2,12 +2,12 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.state +package io.airbyte.cdk.load.state import edu.umd.cs.findbugs.annotations.SuppressFBWarnings -import io.airbyte.cdk.command.DestinationCatalog -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.write.StreamLoader +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.write.StreamLoader import io.micronaut.context.annotation.Factory import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/DestinationTaskExceptionHandler.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandler.kt similarity index 92% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/DestinationTaskExceptionHandler.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandler.kt index 889bb51b079..0712eb0f34f 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/DestinationTaskExceptionHandler.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandler.kt @@ -2,16 +2,16 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task +package io.airbyte.cdk.load.task import edu.umd.cs.findbugs.annotations.SuppressFBWarnings -import io.airbyte.cdk.command.DestinationCatalog -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.state.StreamSucceeded -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.state.SyncSuccess -import io.airbyte.cdk.task.implementor.FailStreamTaskFactory -import io.airbyte.cdk.task.implementor.FailSyncTaskFactory +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.state.StreamSucceeded +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.state.SyncSuccess +import io.airbyte.cdk.load.task.implementor.FailStreamTaskFactory +import io.airbyte.cdk.load.task.implementor.FailSyncTaskFactory import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/DestinationTaskLauncher.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncher.kt similarity index 87% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/DestinationTaskLauncher.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncher.kt index df989c67c1e..efcb9d8ed50 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/DestinationTaskLauncher.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncher.kt @@ -2,26 +2,26 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task +package io.airbyte.cdk.load.task import edu.umd.cs.findbugs.annotations.SuppressFBWarnings -import io.airbyte.cdk.command.DestinationCatalog -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.message.Batch -import io.airbyte.cdk.message.BatchEnvelope -import io.airbyte.cdk.message.SpilledRawMessagesLocalFile -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.task.implementor.CloseStreamTaskFactory -import io.airbyte.cdk.task.implementor.OpenStreamTaskFactory -import io.airbyte.cdk.task.implementor.ProcessBatchTaskFactory -import io.airbyte.cdk.task.implementor.ProcessRecordsTaskFactory -import io.airbyte.cdk.task.implementor.SetupTaskFactory -import io.airbyte.cdk.task.implementor.TeardownTaskFactory -import io.airbyte.cdk.task.internal.FlushCheckpointsTaskFactory -import io.airbyte.cdk.task.internal.InputConsumerTask -import io.airbyte.cdk.task.internal.SpillToDiskTaskFactory -import io.airbyte.cdk.task.internal.TimedForcedCheckpointFlushTask -import io.airbyte.cdk.task.internal.UpdateCheckpointsTask +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.message.Batch +import io.airbyte.cdk.load.message.BatchEnvelope +import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.task.implementor.CloseStreamTaskFactory +import io.airbyte.cdk.load.task.implementor.OpenStreamTaskFactory +import io.airbyte.cdk.load.task.implementor.ProcessBatchTaskFactory +import io.airbyte.cdk.load.task.implementor.ProcessRecordsTaskFactory +import io.airbyte.cdk.load.task.implementor.SetupTaskFactory +import io.airbyte.cdk.load.task.implementor.TeardownTaskFactory +import io.airbyte.cdk.load.task.internal.FlushCheckpointsTaskFactory +import io.airbyte.cdk.load.task.internal.InputConsumerTask +import io.airbyte.cdk.load.task.internal.SpillToDiskTaskFactory +import io.airbyte.cdk.load.task.internal.TimedForcedCheckpointFlushTask +import io.airbyte.cdk.load.task.internal.UpdateCheckpointsTask import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/DestinationTaskScopeProvider.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskScopeProvider.kt similarity index 97% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/DestinationTaskScopeProvider.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskScopeProvider.kt index 39778e2642e..bdfd7e16a14 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/DestinationTaskScopeProvider.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskScopeProvider.kt @@ -2,9 +2,9 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task +package io.airbyte.cdk.load.task -import io.airbyte.cdk.command.DestinationConfiguration +import io.airbyte.cdk.load.command.DestinationConfiguration import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/Task.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/Task.kt similarity index 89% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/Task.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/Task.kt index 53e356eefba..df9c6fc1b0e 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/Task.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/Task.kt @@ -2,9 +2,9 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task +package io.airbyte.cdk.load.task -import io.airbyte.cdk.util.CloseableCoroutine +import io.airbyte.cdk.load.util.CloseableCoroutine interface Task { suspend fun execute() diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/CloseStreamTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/CloseStreamTask.kt similarity index 79% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/CloseStreamTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/CloseStreamTask.kt index 20ca9bc7627..c99aa8385a8 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/CloseStreamTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/CloseStreamTask.kt @@ -2,14 +2,14 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.implementor +package io.airbyte.cdk.load.task.implementor -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.task.DestinationTaskLauncher -import io.airbyte.cdk.task.ImplementorTask -import io.airbyte.cdk.task.StreamTask -import io.airbyte.cdk.write.StreamLoader +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.task.DestinationTaskLauncher +import io.airbyte.cdk.load.task.ImplementorTask +import io.airbyte.cdk.load.task.StreamTask +import io.airbyte.cdk.load.write.StreamLoader import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/FailStreamTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/FailStreamTask.kt similarity index 89% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/FailStreamTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/FailStreamTask.kt index 2e7f027734b..404cdf961f1 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/FailStreamTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/FailStreamTask.kt @@ -2,13 +2,13 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.implementor +package io.airbyte.cdk.load.task.implementor -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.state.StreamIncompleteResult -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.task.DestinationTaskExceptionHandler -import io.airbyte.cdk.task.ImplementorTask +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.state.StreamIncompleteResult +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.task.DestinationTaskExceptionHandler +import io.airbyte.cdk.load.task.ImplementorTask import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/FailSyncTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/FailSyncTask.kt similarity index 85% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/FailSyncTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/FailSyncTask.kt index 5cd10ab7f3d..fb87c84cafe 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/FailSyncTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/FailSyncTask.kt @@ -2,13 +2,13 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.implementor +package io.airbyte.cdk.load.task.implementor -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.task.DestinationTaskExceptionHandler -import io.airbyte.cdk.task.ImplementorTask -import io.airbyte.cdk.util.setOnce -import io.airbyte.cdk.write.DestinationWriter +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.task.DestinationTaskExceptionHandler +import io.airbyte.cdk.load.task.ImplementorTask +import io.airbyte.cdk.load.util.setOnce +import io.airbyte.cdk.load.write.DestinationWriter import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/OpenStreamTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/OpenStreamTask.kt similarity index 78% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/OpenStreamTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/OpenStreamTask.kt index ce70140b51f..ba996bcde43 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/OpenStreamTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/OpenStreamTask.kt @@ -2,15 +2,15 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.implementor +package io.airbyte.cdk.load.task.implementor -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.task.DestinationTaskLauncher -import io.airbyte.cdk.task.ImplementorTask -import io.airbyte.cdk.task.StreamTask -import io.airbyte.cdk.write.DestinationWriter -import io.airbyte.cdk.write.StreamLoader +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.task.DestinationTaskLauncher +import io.airbyte.cdk.load.task.ImplementorTask +import io.airbyte.cdk.load.task.StreamTask +import io.airbyte.cdk.load.write.DestinationWriter +import io.airbyte.cdk.load.write.StreamLoader import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/ProcessBatchTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/ProcessBatchTask.kt similarity index 78% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/ProcessBatchTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/ProcessBatchTask.kt index 7b319256c3e..2b959d43ffb 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/ProcessBatchTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/ProcessBatchTask.kt @@ -2,15 +2,15 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.implementor +package io.airbyte.cdk.load.task.implementor -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.message.BatchEnvelope -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.task.DestinationTaskLauncher -import io.airbyte.cdk.task.ImplementorTask -import io.airbyte.cdk.task.StreamTask -import io.airbyte.cdk.write.StreamLoader +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.message.BatchEnvelope +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.task.DestinationTaskLauncher +import io.airbyte.cdk.load.task.ImplementorTask +import io.airbyte.cdk.load.task.StreamTask +import io.airbyte.cdk.load.write.StreamLoader import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/ProcessRecordsTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTask.kt similarity index 81% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/ProcessRecordsTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTask.kt index d642e6a4900..2ce758b5f3f 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/ProcessRecordsTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTask.kt @@ -2,22 +2,22 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.implementor +package io.airbyte.cdk.load.task.implementor -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.message.BatchEnvelope -import io.airbyte.cdk.message.Deserializer -import io.airbyte.cdk.message.DestinationMessage -import io.airbyte.cdk.message.DestinationRecord -import io.airbyte.cdk.message.DestinationStreamAffinedMessage -import io.airbyte.cdk.message.DestinationStreamComplete -import io.airbyte.cdk.message.DestinationStreamIncomplete -import io.airbyte.cdk.message.SpilledRawMessagesLocalFile -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.task.DestinationTaskLauncher -import io.airbyte.cdk.task.ImplementorTask -import io.airbyte.cdk.task.StreamTask -import io.airbyte.cdk.write.StreamLoader +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.message.BatchEnvelope +import io.airbyte.cdk.load.message.Deserializer +import io.airbyte.cdk.load.message.DestinationMessage +import io.airbyte.cdk.load.message.DestinationRecord +import io.airbyte.cdk.load.message.DestinationStreamAffinedMessage +import io.airbyte.cdk.load.message.DestinationStreamComplete +import io.airbyte.cdk.load.message.DestinationStreamIncomplete +import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.task.DestinationTaskLauncher +import io.airbyte.cdk.load.task.ImplementorTask +import io.airbyte.cdk.load.task.StreamTask +import io.airbyte.cdk.load.write.StreamLoader import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/SetupTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/SetupTask.kt similarity index 80% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/SetupTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/SetupTask.kt index b6a48a4174b..33d949ea1f9 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/SetupTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/SetupTask.kt @@ -2,12 +2,12 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.implementor +package io.airbyte.cdk.load.task.implementor -import io.airbyte.cdk.task.DestinationTaskLauncher -import io.airbyte.cdk.task.ImplementorTask -import io.airbyte.cdk.task.SyncTask -import io.airbyte.cdk.write.DestinationWriter +import io.airbyte.cdk.load.task.DestinationTaskLauncher +import io.airbyte.cdk.load.task.ImplementorTask +import io.airbyte.cdk.load.task.SyncTask +import io.airbyte.cdk.load.write.DestinationWriter import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/TeardownTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/TeardownTask.kt similarity index 88% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/TeardownTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/TeardownTask.kt index 7455a13eeb3..1a361deb0af 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/implementor/TeardownTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/TeardownTask.kt @@ -2,13 +2,13 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.implementor +package io.airbyte.cdk.load.task.implementor -import io.airbyte.cdk.state.CheckpointManager -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.task.DestinationTaskLauncher -import io.airbyte.cdk.task.SyncTask -import io.airbyte.cdk.write.DestinationWriter +import io.airbyte.cdk.load.state.CheckpointManager +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.task.DestinationTaskLauncher +import io.airbyte.cdk.load.task.SyncTask +import io.airbyte.cdk.load.write.DestinationWriter import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/FlushCheckpointsTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/FlushCheckpointsTask.kt similarity index 81% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/FlushCheckpointsTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/FlushCheckpointsTask.kt index 9484bcbad0c..18273d9242f 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/FlushCheckpointsTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/FlushCheckpointsTask.kt @@ -2,11 +2,11 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.internal +package io.airbyte.cdk.load.task.internal -import io.airbyte.cdk.state.CheckpointManager -import io.airbyte.cdk.task.InternalTask -import io.airbyte.cdk.task.SyncTask +import io.airbyte.cdk.load.state.CheckpointManager +import io.airbyte.cdk.load.task.InternalTask +import io.airbyte.cdk.load.task.SyncTask import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/InputConsumerTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/InputConsumerTask.kt similarity index 83% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/InputConsumerTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/InputConsumerTask.kt index 249515121cb..27d2e7f7264 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/InputConsumerTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/InputConsumerTask.kt @@ -2,36 +2,36 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.internal +package io.airbyte.cdk.load.task.internal import edu.umd.cs.findbugs.annotations.SuppressFBWarnings -import io.airbyte.cdk.command.DestinationCatalog -import io.airbyte.cdk.command.DestinationConfiguration -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.message.CheckpointMessage -import io.airbyte.cdk.message.CheckpointMessageWrapped -import io.airbyte.cdk.message.Deserializer -import io.airbyte.cdk.message.DestinationMessage -import io.airbyte.cdk.message.DestinationRecord -import io.airbyte.cdk.message.DestinationRecordWrapped -import io.airbyte.cdk.message.DestinationStreamAffinedMessage -import io.airbyte.cdk.message.DestinationStreamComplete -import io.airbyte.cdk.message.DestinationStreamIncomplete -import io.airbyte.cdk.message.GlobalCheckpoint -import io.airbyte.cdk.message.GlobalCheckpointWrapped -import io.airbyte.cdk.message.MessageQueueSupplier -import io.airbyte.cdk.message.QueueWriter -import io.airbyte.cdk.message.StreamCheckpoint -import io.airbyte.cdk.message.StreamCheckpointWrapped -import io.airbyte.cdk.message.StreamCompleteWrapped -import io.airbyte.cdk.message.StreamRecordWrapped -import io.airbyte.cdk.message.Undefined -import io.airbyte.cdk.state.MemoryManager -import io.airbyte.cdk.state.Reserved -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.task.InternalTask -import io.airbyte.cdk.task.SyncTask -import io.airbyte.cdk.util.use +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.command.DestinationConfiguration +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.message.CheckpointMessage +import io.airbyte.cdk.load.message.CheckpointMessageWrapped +import io.airbyte.cdk.load.message.Deserializer +import io.airbyte.cdk.load.message.DestinationMessage +import io.airbyte.cdk.load.message.DestinationRecord +import io.airbyte.cdk.load.message.DestinationRecordWrapped +import io.airbyte.cdk.load.message.DestinationStreamAffinedMessage +import io.airbyte.cdk.load.message.DestinationStreamComplete +import io.airbyte.cdk.load.message.DestinationStreamIncomplete +import io.airbyte.cdk.load.message.GlobalCheckpoint +import io.airbyte.cdk.load.message.GlobalCheckpointWrapped +import io.airbyte.cdk.load.message.MessageQueueSupplier +import io.airbyte.cdk.load.message.QueueWriter +import io.airbyte.cdk.load.message.StreamCheckpoint +import io.airbyte.cdk.load.message.StreamCheckpointWrapped +import io.airbyte.cdk.load.message.StreamCompleteWrapped +import io.airbyte.cdk.load.message.StreamRecordWrapped +import io.airbyte.cdk.load.message.Undefined +import io.airbyte.cdk.load.state.MemoryManager +import io.airbyte.cdk.load.state.Reserved +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.task.InternalTask +import io.airbyte.cdk.load.task.SyncTask +import io.airbyte.cdk.load.util.use import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/SpillToDiskTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTask.kt similarity index 81% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/SpillToDiskTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTask.kt index b24de32478a..7eddd90c250 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/SpillToDiskTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTask.kt @@ -2,27 +2,27 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.internal +package io.airbyte.cdk.load.task.internal import com.google.common.collect.Range -import io.airbyte.cdk.command.DestinationConfiguration -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.file.TempFileProvider -import io.airbyte.cdk.message.BatchEnvelope -import io.airbyte.cdk.message.DestinationRecordWrapped -import io.airbyte.cdk.message.MessageQueueSupplier -import io.airbyte.cdk.message.QueueReader -import io.airbyte.cdk.message.SpilledRawMessagesLocalFile -import io.airbyte.cdk.message.StreamCompleteWrapped -import io.airbyte.cdk.message.StreamRecordWrapped -import io.airbyte.cdk.state.FlushStrategy -import io.airbyte.cdk.state.Reserved -import io.airbyte.cdk.task.DestinationTaskLauncher -import io.airbyte.cdk.task.InternalTask -import io.airbyte.cdk.task.StreamTask -import io.airbyte.cdk.util.takeUntilInclusive -import io.airbyte.cdk.util.use -import io.airbyte.cdk.util.withNextAdjacentValue +import io.airbyte.cdk.load.command.DestinationConfiguration +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.file.TempFileProvider +import io.airbyte.cdk.load.message.BatchEnvelope +import io.airbyte.cdk.load.message.DestinationRecordWrapped +import io.airbyte.cdk.load.message.MessageQueueSupplier +import io.airbyte.cdk.load.message.QueueReader +import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile +import io.airbyte.cdk.load.message.StreamCompleteWrapped +import io.airbyte.cdk.load.message.StreamRecordWrapped +import io.airbyte.cdk.load.state.FlushStrategy +import io.airbyte.cdk.load.state.Reserved +import io.airbyte.cdk.load.task.DestinationTaskLauncher +import io.airbyte.cdk.load.task.InternalTask +import io.airbyte.cdk.load.task.StreamTask +import io.airbyte.cdk.load.util.takeUntilInclusive +import io.airbyte.cdk.load.util.use +import io.airbyte.cdk.load.util.withNextAdjacentValue import io.github.oshai.kotlinlogging.KotlinLogging import jakarta.inject.Singleton import kotlinx.coroutines.flow.last diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/TimedForcedCheckpointFlushTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/TimedForcedCheckpointFlushTask.kt similarity index 84% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/TimedForcedCheckpointFlushTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/TimedForcedCheckpointFlushTask.kt index 335b4fcb391..cfeb2048163 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/TimedForcedCheckpointFlushTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/TimedForcedCheckpointFlushTask.kt @@ -2,16 +2,16 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.internal +package io.airbyte.cdk.load.task.internal -import io.airbyte.cdk.command.DestinationConfiguration -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.file.TimeProvider -import io.airbyte.cdk.message.ChannelMessageQueue -import io.airbyte.cdk.message.QueueWriter -import io.airbyte.cdk.state.CheckpointManager -import io.airbyte.cdk.task.SyncTask -import io.airbyte.cdk.util.use +import io.airbyte.cdk.load.command.DestinationConfiguration +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.file.TimeProvider +import io.airbyte.cdk.load.message.ChannelMessageQueue +import io.airbyte.cdk.load.message.QueueWriter +import io.airbyte.cdk.load.state.CheckpointManager +import io.airbyte.cdk.load.task.SyncTask +import io.airbyte.cdk.load.util.use import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/UpdateCheckpointsTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/UpdateCheckpointsTask.kt similarity index 72% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/UpdateCheckpointsTask.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/UpdateCheckpointsTask.kt index d81a167df75..949cc8fed03 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/task/internal/UpdateCheckpointsTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/UpdateCheckpointsTask.kt @@ -2,17 +2,17 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.internal +package io.airbyte.cdk.load.task.internal -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.message.CheckpointMessage -import io.airbyte.cdk.message.CheckpointMessageWrapped -import io.airbyte.cdk.message.GlobalCheckpointWrapped -import io.airbyte.cdk.message.MessageQueue -import io.airbyte.cdk.message.StreamCheckpointWrapped -import io.airbyte.cdk.state.CheckpointManager -import io.airbyte.cdk.state.Reserved -import io.airbyte.cdk.task.SyncTask +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.message.CheckpointMessage +import io.airbyte.cdk.load.message.CheckpointMessageWrapped +import io.airbyte.cdk.load.message.GlobalCheckpointWrapped +import io.airbyte.cdk.load.message.MessageQueue +import io.airbyte.cdk.load.message.StreamCheckpointWrapped +import io.airbyte.cdk.load.state.CheckpointManager +import io.airbyte.cdk.load.state.Reserved +import io.airbyte.cdk.load.task.SyncTask import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/util/CoroutineUtils.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/CoroutineUtils.kt similarity index 95% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/util/CoroutineUtils.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/CoroutineUtils.kt index e6d916a0cef..4644b29f886 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/util/CoroutineUtils.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/CoroutineUtils.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.util +package io.airbyte.cdk.load.util import java.util.concurrent.atomic.AtomicBoolean import kotlinx.coroutines.flow.Flow diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/util/RangeUtils.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/RangeUtils.kt similarity index 93% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/util/RangeUtils.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/RangeUtils.kt index e17f36334ac..e4d7ec1c858 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/util/RangeUtils.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/RangeUtils.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.util +package io.airbyte.cdk.load.util import com.google.common.collect.Range diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/write/DestinationWriter.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/write/DestinationWriter.kt similarity index 89% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/write/DestinationWriter.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/write/DestinationWriter.kt index 50018395d58..7ad369a3623 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/write/DestinationWriter.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/write/DestinationWriter.kt @@ -2,10 +2,10 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.write +package io.airbyte.cdk.load.write -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.state.SyncFailure +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.state.SyncFailure import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/write/StreamLoader.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/write/StreamLoader.kt similarity index 84% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/write/StreamLoader.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/write/StreamLoader.kt index ed28623a3dd..d7b0581590d 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/write/StreamLoader.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/write/StreamLoader.kt @@ -2,13 +2,13 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.write +package io.airbyte.cdk.load.write -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.message.Batch -import io.airbyte.cdk.message.DestinationRecord -import io.airbyte.cdk.message.SimpleBatch -import io.airbyte.cdk.state.StreamIncompleteResult +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.message.Batch +import io.airbyte.cdk.load.message.DestinationRecord +import io.airbyte.cdk.load.message.SimpleBatch +import io.airbyte.cdk.load.state.StreamIncompleteResult /** * Implementor interface. The framework calls open and close once per stream at the beginning and diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/write/WriteOperation.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/write/WriteOperation.kt similarity index 87% rename from airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/write/WriteOperation.kt rename to airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/write/WriteOperation.kt index 51704b49275..6955da322fb 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/write/WriteOperation.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/write/WriteOperation.kt @@ -2,13 +2,13 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.write +package io.airbyte.cdk.load.write import io.airbyte.cdk.Operation -import io.airbyte.cdk.state.SyncFailure -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.state.SyncSuccess -import io.airbyte.cdk.task.TaskLauncher +import io.airbyte.cdk.load.state.SyncFailure +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.state.SyncSuccess +import io.airbyte.cdk.load.task.TaskLauncher import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Factory import io.micronaut.context.annotation.Requires diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/command/DestinationCatalogTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/command/DestinationCatalogTest.kt similarity index 98% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/command/DestinationCatalogTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/command/DestinationCatalogTest.kt index d6db2d9601c..c21f0775ffe 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/command/DestinationCatalogTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/command/DestinationCatalogTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.command +package io.airbyte.cdk.load.command import io.airbyte.protocol.models.Jsons import io.airbyte.protocol.models.v0.AirbyteStream diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/command/MockDestinationCatalogFactory.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/command/MockDestinationCatalogFactory.kt similarity index 92% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/command/MockDestinationCatalogFactory.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/command/MockDestinationCatalogFactory.kt index bb57923822c..7076f46b81d 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/command/MockDestinationCatalogFactory.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/command/MockDestinationCatalogFactory.kt @@ -2,12 +2,12 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.command +package io.airbyte.cdk.load.command -import io.airbyte.cdk.data.FieldType -import io.airbyte.cdk.data.IntegerType -import io.airbyte.cdk.data.ObjectType -import io.airbyte.cdk.data.StringType +import io.airbyte.cdk.load.data.FieldType +import io.airbyte.cdk.load.data.IntegerType +import io.airbyte.cdk.load.data.ObjectType +import io.airbyte.cdk.load.data.StringType import io.micronaut.context.annotation.Factory import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/command/MockDestinationConfiguration.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/command/MockDestinationConfiguration.kt similarity index 95% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/command/MockDestinationConfiguration.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/command/MockDestinationConfiguration.kt index 8a7efd13bf5..a6e0c39c234 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/command/MockDestinationConfiguration.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/command/MockDestinationConfiguration.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.command +package io.airbyte.cdk.load.command import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/data/AirbyteTypeToJsonSchemaTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/data/AirbyteSchemaTypeToJsonSchemaTest.kt similarity index 98% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/data/AirbyteTypeToJsonSchemaTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/data/AirbyteSchemaTypeToJsonSchemaTest.kt index 5005755fcda..52011496f2b 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/data/AirbyteTypeToJsonSchemaTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/data/AirbyteSchemaTypeToJsonSchemaTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.data +package io.airbyte.cdk.load.data import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.JsonNodeFactory @@ -10,7 +10,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test -class AirbyteTypeToJsonSchemaTest { +class AirbyteSchemaTypeToJsonSchemaTest { @Test fun testRoundTrip() { val schema = JsonNodeFactory.instance.objectNode() diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/data/AirbyteValueToJsonTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/data/AirbyteValueToJsonTest.kt similarity index 99% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/data/AirbyteValueToJsonTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/data/AirbyteValueToJsonTest.kt index dedc0997a8d..e74922f2430 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/data/AirbyteValueToJsonTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/data/AirbyteValueToJsonTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.data +package io.airbyte.cdk.load.data import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/data/JsonSchemaToAirbyteTypeTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/data/JsonSchemaToAirbyteSchemaTypeTest.kt similarity index 99% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/data/JsonSchemaToAirbyteTypeTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/data/JsonSchemaToAirbyteSchemaTypeTest.kt index bea239ed17f..24847b151e0 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/data/JsonSchemaToAirbyteTypeTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/data/JsonSchemaToAirbyteSchemaTypeTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.data +package io.airbyte.cdk.load.data import com.fasterxml.jackson.databind.node.JsonNodeFactory import com.fasterxml.jackson.databind.node.ObjectNode @@ -10,7 +10,7 @@ import io.airbyte.cdk.util.Jsons import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test -class JsonSchemaToAirbyteTypeTest { +class JsonSchemaToAirbyteSchemaTypeTest { private fun ofType(type: String): ObjectNode { return JsonNodeFactory.instance.objectNode().put("type", type) } diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/data/JsonToAirbyteValueTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/data/JsonToAirbyteValueTest.kt similarity index 99% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/data/JsonToAirbyteValueTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/data/JsonToAirbyteValueTest.kt index 38b3259652b..225b8d12720 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/data/JsonToAirbyteValueTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/data/JsonToAirbyteValueTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.data +package io.airbyte.cdk.load.data import com.fasterxml.jackson.databind.node.JsonNodeFactory import java.math.BigDecimal diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/file/MockTempFileProvider.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/file/MockTempFileProvider.kt similarity index 98% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/file/MockTempFileProvider.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/file/MockTempFileProvider.kt index ff844ec7670..093e160f312 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/file/MockTempFileProvider.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/file/MockTempFileProvider.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.file +package io.airbyte.cdk.load.file import io.micronaut.context.annotation.Requires import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/file/MockTimeProvider.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/file/MockTimeProvider.kt similarity index 95% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/file/MockTimeProvider.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/file/MockTimeProvider.kt index 052efeec6f0..7909f2bfdf2 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/file/MockTimeProvider.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/file/MockTimeProvider.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.file +package io.airbyte.cdk.load.file import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/message/DestinationMessageTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/message/DestinationMessageTest.kt similarity index 95% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/message/DestinationMessageTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/message/DestinationMessageTest.kt index c7ba1edb000..10b0ea1d89f 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/message/DestinationMessageTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/message/DestinationMessageTest.kt @@ -2,12 +2,12 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.message +package io.airbyte.cdk.load.message -import io.airbyte.cdk.command.Append -import io.airbyte.cdk.command.DestinationCatalog -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.data.ObjectTypeWithEmptySchema +import io.airbyte.cdk.load.command.Append +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.data.ObjectTypeWithEmptySchema import io.airbyte.protocol.models.Jsons import io.airbyte.protocol.models.v0.AirbyteGlobalState import io.airbyte.protocol.models.v0.AirbyteMessage @@ -19,7 +19,7 @@ import io.airbyte.protocol.models.v0.AirbyteStateStats import io.airbyte.protocol.models.v0.AirbyteStreamState import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage import io.airbyte.protocol.models.v0.AirbyteTraceMessage -import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments @@ -47,7 +47,7 @@ class DestinationMessageTest { fun testRoundTrip(message: AirbyteMessage) { val roundTripped = factory.fromAirbyteMessage(message, Jsons.serialize(message)).asProtocolMessage() - assertEquals(message, roundTripped) + Assertions.assertEquals(message, roundTripped) } // Checkpoint messages aren't round-trippable. @@ -73,7 +73,7 @@ class DestinationMessageTest { factory.fromAirbyteMessage(inputMessage, Jsons.serialize(inputMessage)) as StreamCheckpoint - assertEquals( + Assertions.assertEquals( inputMessage.also { it.state.destinationStats = AirbyteStateStats().withRecordCount(3.0) }, @@ -110,7 +110,7 @@ class DestinationMessageTest { Jsons.serialize(inputMessage), ) as GlobalCheckpoint - assertEquals( + Assertions.assertEquals( inputMessage.also { it.state.destinationStats = AirbyteStateStats().withRecordCount(3.0) }, diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/CheckpointManagerTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/CheckpointManagerTest.kt similarity index 98% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/CheckpointManagerTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/CheckpointManagerTest.kt index 40a409c79e4..2b8201112bf 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/CheckpointManagerTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/CheckpointManagerTest.kt @@ -2,18 +2,18 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.state +package io.airbyte.cdk.load.state import com.google.common.collect.Range import com.google.common.collect.TreeRangeSet -import io.airbyte.cdk.command.DestinationCatalog -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1 -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream2 -import io.airbyte.cdk.file.TimeProvider -import io.airbyte.cdk.message.Batch -import io.airbyte.cdk.message.BatchEnvelope -import io.airbyte.cdk.message.SimpleBatch +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream1 +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream2 +import io.airbyte.cdk.load.file.TimeProvider +import io.airbyte.cdk.load.message.Batch +import io.airbyte.cdk.load.message.BatchEnvelope +import io.airbyte.cdk.load.message.SimpleBatch import io.micronaut.context.annotation.Requires import io.micronaut.test.extensions.junit5.annotation.MicronautTest import jakarta.inject.Inject diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/DefaultFlushStrategyTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/DefaultFlushStrategyTest.kt similarity index 91% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/DefaultFlushStrategyTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/DefaultFlushStrategyTest.kt index 5b2b640f6b7..cc277e5e3ce 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/DefaultFlushStrategyTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/DefaultFlushStrategyTest.kt @@ -2,14 +2,14 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.state +package io.airbyte.cdk.load.state import com.google.common.collect.Range -import io.airbyte.cdk.command.DestinationConfiguration -import io.airbyte.cdk.command.MockDestinationCatalogFactory -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream2 -import io.airbyte.cdk.message.ChannelMessageQueue -import io.airbyte.cdk.task.internal.ForceFlushEvent +import io.airbyte.cdk.load.command.DestinationConfiguration +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream2 +import io.airbyte.cdk.load.message.ChannelMessageQueue +import io.airbyte.cdk.load.task.internal.ForceFlushEvent import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires import io.micronaut.test.extensions.junit5.annotation.MicronautTest diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/MemoryManagerTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/MemoryManagerTest.kt similarity index 99% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/MemoryManagerTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/MemoryManagerTest.kt index cf53c478df3..b2c66db6eea 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/MemoryManagerTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/MemoryManagerTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.state +package io.airbyte.cdk.load.state import io.micronaut.context.annotation.Replaces import io.micronaut.context.annotation.Requires diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/MockCheckpointManager.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/MockCheckpointManager.kt similarity index 90% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/MockCheckpointManager.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/MockCheckpointManager.kt index 002ae813401..ec2920fdeb9 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/MockCheckpointManager.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/MockCheckpointManager.kt @@ -2,11 +2,11 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.state +package io.airbyte.cdk.load.state -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.file.TimeProvider -import io.airbyte.cdk.message.CheckpointMessage +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.file.TimeProvider +import io.airbyte.cdk.load.message.CheckpointMessage import io.micronaut.context.annotation.Requires import jakarta.inject.Inject import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/StreamManagerTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/StreamManagerTest.kt similarity index 96% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/StreamManagerTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/StreamManagerTest.kt index 727041adbcc..ceb8f009fb8 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/StreamManagerTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/StreamManagerTest.kt @@ -2,15 +2,15 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.state +package io.airbyte.cdk.load.state import com.google.common.collect.Range -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1 -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream2 -import io.airbyte.cdk.message.Batch -import io.airbyte.cdk.message.BatchEnvelope -import io.airbyte.cdk.message.SimpleBatch +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream1 +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream2 +import io.airbyte.cdk.load.message.Batch +import io.airbyte.cdk.load.message.BatchEnvelope +import io.airbyte.cdk.load.message.SimpleBatch import java.util.stream.Stream import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.delay diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/SyncManagerTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/SyncManagerTest.kt similarity index 93% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/SyncManagerTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/SyncManagerTest.kt index ea017a0f934..8453f5d7670 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/SyncManagerTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/SyncManagerTest.kt @@ -2,12 +2,12 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.state +package io.airbyte.cdk.load.state -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1 -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream2 -import io.airbyte.cdk.test.util.CoroutineTestUtils +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream1 +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream2 +import io.airbyte.cdk.load.test.util.CoroutineTestUtils import io.micronaut.test.extensions.junit5.annotation.MicronautTest import jakarta.inject.Inject import kotlinx.coroutines.channels.Channel diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/SyncManagerUtils.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/SyncManagerUtils.kt similarity index 75% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/SyncManagerUtils.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/SyncManagerUtils.kt index ea57af85f96..28a24a2b836 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/state/SyncManagerUtils.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/state/SyncManagerUtils.kt @@ -2,13 +2,13 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.state +package io.airbyte.cdk.load.state import com.google.common.collect.Range -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.message.Batch -import io.airbyte.cdk.message.BatchEnvelope -import io.airbyte.cdk.message.SimpleBatch +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.message.Batch +import io.airbyte.cdk.load.message.BatchEnvelope +import io.airbyte.cdk.load.message.SimpleBatch /** * Because [SyncManager] and [StreamManager] have thin interfaces with no side effects, mocking them diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/DestinationTaskExceptionHandlerTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandlerTest.kt similarity index 87% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/DestinationTaskExceptionHandlerTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandlerTest.kt index a80559d0062..7760debf537 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/DestinationTaskExceptionHandlerTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandlerTest.kt @@ -2,17 +2,17 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task +package io.airbyte.cdk.load.task -import io.airbyte.cdk.command.DestinationCatalog -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1 -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.task.implementor.FailStreamTask -import io.airbyte.cdk.task.implementor.FailStreamTaskFactory -import io.airbyte.cdk.task.implementor.FailSyncTask -import io.airbyte.cdk.task.implementor.FailSyncTaskFactory -import io.airbyte.cdk.test.util.CoroutineTestUtils +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.task.implementor.FailStreamTask +import io.airbyte.cdk.load.task.implementor.FailStreamTaskFactory +import io.airbyte.cdk.load.task.implementor.FailSyncTask +import io.airbyte.cdk.load.task.implementor.FailSyncTaskFactory +import io.airbyte.cdk.load.test.util.CoroutineTestUtils import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires import io.micronaut.test.extensions.junit5.annotation.MicronautTest @@ -87,7 +87,7 @@ class DestinationTaskExceptionHandlerTest { runTest { val mockTask = object : StreamTask { - override val stream: DestinationStream = stream1 + override val stream: DestinationStream = MockDestinationCatalogFactory.stream1 override suspend fun execute() { throw RuntimeException("StreamTask failure") @@ -97,7 +97,7 @@ class DestinationTaskExceptionHandlerTest { val wrappedTask = exceptionHandler.withExceptionHandling(mockTask) wrappedTask.execute() val (stream, exception) = mockFailStreamTaskFactory.didRunFor.receive() - Assertions.assertEquals(stream1, stream) + Assertions.assertEquals(MockDestinationCatalogFactory.stream1, stream) Assertions.assertTrue(exception is RuntimeException) } @@ -182,7 +182,7 @@ class DestinationTaskExceptionHandlerTest { val innerTaskRan = Channel(Channel.UNLIMITED) val mockTask = object : StreamTask { - override val stream: DestinationStream = stream1 + override val stream: DestinationStream = MockDestinationCatalogFactory.stream1 override suspend fun execute() { innerTaskRan.send(true) @@ -190,7 +190,7 @@ class DestinationTaskExceptionHandlerTest { } val wrappedTask = exceptionHandler.withExceptionHandling(mockTask) - val manager = syncManager.getStreamManager(stream1.descriptor) + val manager = syncManager.getStreamManager(MockDestinationCatalogFactory.stream1.descriptor) manager.markEndOfStream() manager.markFailed(RuntimeException("dummy failure")) launch { wrappedTask.execute() } @@ -207,7 +207,7 @@ class DestinationTaskExceptionHandlerTest { ) = runTest { val mockTask = object : StreamTask { - override val stream: DestinationStream = stream1 + override val stream: DestinationStream = MockDestinationCatalogFactory.stream1 override suspend fun execute() { // do nothing @@ -216,7 +216,7 @@ class DestinationTaskExceptionHandlerTest { val wrappedTask = exceptionHandler.withExceptionHandling(mockTask) - val manager = syncManager.getStreamManager(stream1.descriptor) + val manager = syncManager.getStreamManager(MockDestinationCatalogFactory.stream1.descriptor) manager.markEndOfStream() manager.markSucceeded() diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/DestinationTaskLauncherTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncherTest.kt similarity index 81% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/DestinationTaskLauncherTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncherTest.kt index 7c045debd7d..eb4f9c4f269 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/DestinationTaskLauncherTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncherTest.kt @@ -2,44 +2,44 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task +package io.airbyte.cdk.load.task import com.google.common.collect.Range import com.google.common.collect.TreeRangeSet -import io.airbyte.cdk.command.DestinationCatalog -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1 -import io.airbyte.cdk.file.DefaultLocalFile -import io.airbyte.cdk.message.Batch -import io.airbyte.cdk.message.BatchEnvelope -import io.airbyte.cdk.message.SpilledRawMessagesLocalFile -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.task.implementor.CloseStreamTask -import io.airbyte.cdk.task.implementor.CloseStreamTaskFactory -import io.airbyte.cdk.task.implementor.DefaultCloseStreamTaskFactory -import io.airbyte.cdk.task.implementor.DefaultOpenStreamTaskFactory -import io.airbyte.cdk.task.implementor.DefaultProcessBatchTaskFactory -import io.airbyte.cdk.task.implementor.DefaultProcessRecordsTaskFactory -import io.airbyte.cdk.task.implementor.DefaultSetupTaskFactory -import io.airbyte.cdk.task.implementor.DefaultTeardownTaskFactory -import io.airbyte.cdk.task.implementor.OpenStreamTask -import io.airbyte.cdk.task.implementor.OpenStreamTaskFactory -import io.airbyte.cdk.task.implementor.ProcessBatchTask -import io.airbyte.cdk.task.implementor.ProcessBatchTaskFactory -import io.airbyte.cdk.task.implementor.ProcessRecordsTask -import io.airbyte.cdk.task.implementor.ProcessRecordsTaskFactory -import io.airbyte.cdk.task.implementor.SetupTask -import io.airbyte.cdk.task.implementor.SetupTaskFactory -import io.airbyte.cdk.task.implementor.TeardownTask -import io.airbyte.cdk.task.implementor.TeardownTaskFactory -import io.airbyte.cdk.task.internal.DefaultSpillToDiskTaskFactory -import io.airbyte.cdk.task.internal.FlushCheckpointsTask -import io.airbyte.cdk.task.internal.FlushCheckpointsTaskFactory -import io.airbyte.cdk.task.internal.InputConsumerTask -import io.airbyte.cdk.task.internal.SpillToDiskTask -import io.airbyte.cdk.task.internal.SpillToDiskTaskFactory -import io.airbyte.cdk.task.internal.TimedForcedCheckpointFlushTask -import io.airbyte.cdk.task.internal.UpdateCheckpointsTask +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory +import io.airbyte.cdk.load.file.DefaultLocalFile +import io.airbyte.cdk.load.message.Batch +import io.airbyte.cdk.load.message.BatchEnvelope +import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.task.implementor.CloseStreamTask +import io.airbyte.cdk.load.task.implementor.CloseStreamTaskFactory +import io.airbyte.cdk.load.task.implementor.DefaultCloseStreamTaskFactory +import io.airbyte.cdk.load.task.implementor.DefaultOpenStreamTaskFactory +import io.airbyte.cdk.load.task.implementor.DefaultProcessBatchTaskFactory +import io.airbyte.cdk.load.task.implementor.DefaultProcessRecordsTaskFactory +import io.airbyte.cdk.load.task.implementor.DefaultSetupTaskFactory +import io.airbyte.cdk.load.task.implementor.DefaultTeardownTaskFactory +import io.airbyte.cdk.load.task.implementor.OpenStreamTask +import io.airbyte.cdk.load.task.implementor.OpenStreamTaskFactory +import io.airbyte.cdk.load.task.implementor.ProcessBatchTask +import io.airbyte.cdk.load.task.implementor.ProcessBatchTaskFactory +import io.airbyte.cdk.load.task.implementor.ProcessRecordsTask +import io.airbyte.cdk.load.task.implementor.ProcessRecordsTaskFactory +import io.airbyte.cdk.load.task.implementor.SetupTask +import io.airbyte.cdk.load.task.implementor.SetupTaskFactory +import io.airbyte.cdk.load.task.implementor.TeardownTask +import io.airbyte.cdk.load.task.implementor.TeardownTaskFactory +import io.airbyte.cdk.load.task.internal.DefaultSpillToDiskTaskFactory +import io.airbyte.cdk.load.task.internal.FlushCheckpointsTask +import io.airbyte.cdk.load.task.internal.FlushCheckpointsTaskFactory +import io.airbyte.cdk.load.task.internal.InputConsumerTask +import io.airbyte.cdk.load.task.internal.SpillToDiskTask +import io.airbyte.cdk.load.task.internal.SpillToDiskTaskFactory +import io.airbyte.cdk.load.task.internal.TimedForcedCheckpointFlushTask +import io.airbyte.cdk.load.task.internal.UpdateCheckpointsTask import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Replaces import io.micronaut.context.annotation.Requires @@ -337,7 +337,7 @@ class DestinationTaskLauncherTest { @Test fun testHandleSpilledFileCompleteNotEndOfStream() = runTest { taskLauncher.handleNewSpilledFile( - stream1, + MockDestinationCatalogFactory.stream1, BatchEnvelope( SpilledRawMessagesLocalFile(DefaultLocalFile(Path("not/a/real/file")), 100L) ), @@ -345,7 +345,8 @@ class DestinationTaskLauncherTest { ) processRecordsTaskFactory.hasRun.receive() - mockSpillToDiskTaskFactory.streamHasRun[stream1.descriptor]?.receive() + mockSpillToDiskTaskFactory.streamHasRun[MockDestinationCatalogFactory.stream1.descriptor] + ?.receive() ?: Assertions.fail("SpillToDiskTask not run") } @@ -353,7 +354,7 @@ class DestinationTaskLauncherTest { fun testHandleSpilledFileCompleteEndOfStream() = runTest { launch { taskLauncher.handleNewSpilledFile( - stream1, + MockDestinationCatalogFactory.stream1, BatchEnvelope( SpilledRawMessagesLocalFile(DefaultLocalFile(Path("not/a/real/file")), 100L) ), @@ -364,22 +365,25 @@ class DestinationTaskLauncherTest { processRecordsTaskFactory.hasRun.receive() delay(500) Assertions.assertTrue( - mockSpillToDiskTaskFactory.streamHasRun[stream1.descriptor]?.tryReceive()?.isFailure != - false + mockSpillToDiskTaskFactory.streamHasRun[ + MockDestinationCatalogFactory.stream1.descriptor] + ?.tryReceive() + ?.isFailure != false ) } @Test fun testHandleNewBatch() = runTest { val range = TreeRangeSet.create(listOf(Range.closed(0L, 100L))) - val streamManager = syncManager.getStreamManager(stream1.descriptor) + val streamManager = + syncManager.getStreamManager(MockDestinationCatalogFactory.stream1.descriptor) repeat(100) { streamManager.countRecordIn() } streamManager.markEndOfStream() // Verify incomplete batch triggers process batch val incompleteBatch = BatchEnvelope(MockBatch(Batch.State.LOCAL), range) - taskLauncher.handleNewBatch(stream1, incompleteBatch) + taskLauncher.handleNewBatch(MockDestinationCatalogFactory.stream1, incompleteBatch) Assertions.assertFalse(streamManager.areRecordsPersistedUntil(100L)) val batchReceived = processBatchTaskFactory.hasRun.receive() @@ -388,21 +392,21 @@ class DestinationTaskLauncherTest { Assertions.assertTrue(flushCheckpointsTaskFactory.hasRun.tryReceive().isFailure) val persistedBatch = BatchEnvelope(MockBatch(Batch.State.PERSISTED), range) - taskLauncher.handleNewBatch(stream1, persistedBatch) + taskLauncher.handleNewBatch(MockDestinationCatalogFactory.stream1, persistedBatch) Assertions.assertTrue(streamManager.areRecordsPersistedUntil(100L)) Assertions.assertTrue(flushCheckpointsTaskFactory.hasRun.receive()) // Verify complete batch w/o batch processing complete does nothing val halfRange = TreeRangeSet.create(listOf(Range.closed(0L, 50L))) val completeBatchHalf = BatchEnvelope(MockBatch(Batch.State.COMPLETE), halfRange) - taskLauncher.handleNewBatch(stream1, completeBatchHalf) + taskLauncher.handleNewBatch(MockDestinationCatalogFactory.stream1, completeBatchHalf) delay(1000) Assertions.assertTrue(closeStreamTaskFactory.hasRun.tryReceive().isFailure) // Verify complete batch w/ batch processing complete triggers close stream val secondHalf = TreeRangeSet.create(listOf(Range.closed(51L, 100L))) val completingBatch = BatchEnvelope(MockBatch(Batch.State.COMPLETE), secondHalf) - taskLauncher.handleNewBatch(stream1, completingBatch) + taskLauncher.handleNewBatch(MockDestinationCatalogFactory.stream1, completingBatch) closeStreamTaskFactory.hasRun.receive() Assertions.assertTrue(true) } @@ -410,7 +414,7 @@ class DestinationTaskLauncherTest { @Test fun testHandleStreamClosed() = runTest { // This should run teardown unconditionally. - launch { taskLauncher.handleStreamClosed(stream1) } + launch { taskLauncher.handleStreamClosed(MockDestinationCatalogFactory.stream1) } teardownTaskFactory.hasRun.receive() } diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/MockScopeProvider.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/MockScopeProvider.kt similarity index 94% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/MockScopeProvider.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/MockScopeProvider.kt index 2290d513902..453325ab855 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/MockScopeProvider.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/MockScopeProvider.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task +package io.airbyte.cdk.load.task import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/MockTaskLauncher.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/MockTaskLauncher.kt similarity index 86% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/MockTaskLauncher.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/MockTaskLauncher.kt index 3401e795bd2..80ce429fe1a 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/MockTaskLauncher.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/MockTaskLauncher.kt @@ -2,11 +2,11 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task +package io.airbyte.cdk.load.task -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.message.BatchEnvelope -import io.airbyte.cdk.message.SpilledRawMessagesLocalFile +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.message.BatchEnvelope +import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires import jakarta.inject.Singleton diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/implementor/ProcessRecordsTaskTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTaskTest.kt similarity index 79% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/implementor/ProcessRecordsTaskTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTaskTest.kt index 88fb9f479f8..ab7029c9944 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/implementor/ProcessRecordsTaskTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTaskTest.kt @@ -2,22 +2,22 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.implementor +package io.airbyte.cdk.load.task.implementor import com.google.common.collect.Range -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1 -import io.airbyte.cdk.data.IntegerValue -import io.airbyte.cdk.file.MockTempFileProvider -import io.airbyte.cdk.message.Batch -import io.airbyte.cdk.message.BatchEnvelope -import io.airbyte.cdk.message.Deserializer -import io.airbyte.cdk.message.DestinationMessage -import io.airbyte.cdk.message.DestinationRecord -import io.airbyte.cdk.message.SpilledRawMessagesLocalFile -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.task.MockTaskLauncher -import io.airbyte.cdk.write.StreamLoader +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory +import io.airbyte.cdk.load.data.IntegerValue +import io.airbyte.cdk.load.file.MockTempFileProvider +import io.airbyte.cdk.load.message.Batch +import io.airbyte.cdk.load.message.BatchEnvelope +import io.airbyte.cdk.load.message.Deserializer +import io.airbyte.cdk.load.message.DestinationMessage +import io.airbyte.cdk.load.message.DestinationRecord +import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.task.MockTaskLauncher +import io.airbyte.cdk.load.write.StreamLoader import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires import io.micronaut.test.extensions.junit5.annotation.MicronautTest @@ -49,7 +49,7 @@ class ProcessRecordsTaskTest { ) : Batch class MockStreamLoader : StreamLoader { - override val stream: DestinationStream = stream1 + override val stream: DestinationStream = MockDestinationCatalogFactory.stream1 data class SumAndCount(val sum: Long = 0, val count: Long = 0) @@ -78,7 +78,7 @@ class ProcessRecordsTaskTest { class MockDeserializer : Deserializer { override fun deserialize(serialized: String): DestinationMessage { return DestinationRecord( - stream = stream1.descriptor, + stream = MockDestinationCatalogFactory.stream1.descriptor, data = IntegerValue(serialized.toLong()), emittedAtMs = 0L, meta = null, @@ -104,7 +104,7 @@ class ProcessRecordsTaskTest { val task = processRecordsTaskFactory.make( taskLauncher = launcher, - stream = stream1, + stream = MockDestinationCatalogFactory.stream1, fileEnvelope = BatchEnvelope(file, Range.closed(0, 1024)) ) mockFile.linesToRead = (0 until recordCount).map { "$it" }.toMutableList() diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/internal/InputConsumerTaskTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/InputConsumerTaskTest.kt similarity index 65% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/internal/InputConsumerTaskTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/InputConsumerTaskTest.kt index 4096ed177c4..b74aaa4616f 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/internal/InputConsumerTaskTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/InputConsumerTaskTest.kt @@ -2,34 +2,33 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.internal +package io.airbyte.cdk.load.task.internal import com.fasterxml.jackson.databind.node.JsonNodeFactory -import io.airbyte.cdk.command.DestinationConfiguration -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1 -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream2 -import io.airbyte.cdk.data.NullValue -import io.airbyte.cdk.message.CheckpointMessage -import io.airbyte.cdk.message.CheckpointMessageWrapped -import io.airbyte.cdk.message.DestinationMessage -import io.airbyte.cdk.message.DestinationRecord -import io.airbyte.cdk.message.DestinationRecordWrapped -import io.airbyte.cdk.message.DestinationStreamComplete -import io.airbyte.cdk.message.DestinationStreamIncomplete -import io.airbyte.cdk.message.GlobalCheckpoint -import io.airbyte.cdk.message.GlobalCheckpointWrapped -import io.airbyte.cdk.message.MessageQueue -import io.airbyte.cdk.message.MessageQueueSupplier -import io.airbyte.cdk.message.StreamCheckpoint -import io.airbyte.cdk.message.StreamCheckpointWrapped -import io.airbyte.cdk.message.StreamCompleteWrapped -import io.airbyte.cdk.message.StreamRecordWrapped -import io.airbyte.cdk.state.MemoryManager -import io.airbyte.cdk.state.Reserved -import io.airbyte.cdk.state.SyncManager -import io.airbyte.cdk.test.util.CoroutineTestUtils -import io.airbyte.cdk.util.takeUntilInclusive +import io.airbyte.cdk.load.command.DestinationConfiguration +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory +import io.airbyte.cdk.load.data.NullValue +import io.airbyte.cdk.load.message.CheckpointMessage +import io.airbyte.cdk.load.message.CheckpointMessageWrapped +import io.airbyte.cdk.load.message.DestinationMessage +import io.airbyte.cdk.load.message.DestinationRecord +import io.airbyte.cdk.load.message.DestinationRecordWrapped +import io.airbyte.cdk.load.message.DestinationStreamComplete +import io.airbyte.cdk.load.message.DestinationStreamIncomplete +import io.airbyte.cdk.load.message.GlobalCheckpoint +import io.airbyte.cdk.load.message.GlobalCheckpointWrapped +import io.airbyte.cdk.load.message.MessageQueue +import io.airbyte.cdk.load.message.MessageQueueSupplier +import io.airbyte.cdk.load.message.StreamCheckpoint +import io.airbyte.cdk.load.message.StreamCheckpointWrapped +import io.airbyte.cdk.load.message.StreamCompleteWrapped +import io.airbyte.cdk.load.message.StreamRecordWrapped +import io.airbyte.cdk.load.state.MemoryManager +import io.airbyte.cdk.load.state.Reserved +import io.airbyte.cdk.load.state.SyncManager +import io.airbyte.cdk.load.test.util.CoroutineTestUtils +import io.airbyte.cdk.load.util.takeUntilInclusive import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires import io.micronaut.test.extensions.junit5.annotation.MicronautTest @@ -128,13 +127,20 @@ class InputConsumerTaskTest { @Test fun testSendRecords() = runTest { - val queue1 = recordQueueSupplier.get(stream1.descriptor) - val queue2 = recordQueueSupplier.get(stream2.descriptor) + val queue1 = recordQueueSupplier.get(MockDestinationCatalogFactory.stream1.descriptor) + val queue2 = recordQueueSupplier.get(MockDestinationCatalogFactory.stream2.descriptor) - val manager1 = syncManager.getStreamManager(stream1.descriptor) - val manager2 = syncManager.getStreamManager(stream2.descriptor) + val manager1 = + syncManager.getStreamManager(MockDestinationCatalogFactory.stream1.descriptor) + val manager2 = + syncManager.getStreamManager(MockDestinationCatalogFactory.stream2.descriptor) - (0 until 10).forEach { mockInputFlow.addMessage(makeRecord(stream1, "test${it}"), it * 2L) } + (0 until 10).forEach { + mockInputFlow.addMessage( + makeRecord(MockDestinationCatalogFactory.stream1, "test${it}"), + it * 2L + ) + } launch { task.execute() } val messages1 = @@ -148,7 +154,11 @@ class InputConsumerTaskTest { Assertions.assertEquals(10, messages1.size) val expectedRecords = (0 until 10).map { - StreamRecordWrapped(it.toLong(), it * 2L, makeRecord(stream1, "test${it}")) + StreamRecordWrapped( + it.toLong(), + it * 2L, + makeRecord(MockDestinationCatalogFactory.stream1, "test${it}") + ) } Assertions.assertEquals(expectedRecords, messages1.map { it.value }) @@ -165,22 +175,35 @@ class InputConsumerTaskTest { @Test fun testSendEndOfStream() = runTest { - val queue1 = recordQueueSupplier.get(stream1.descriptor) - val queue2 = recordQueueSupplier.get(stream2.descriptor) + val queue1 = recordQueueSupplier.get(MockDestinationCatalogFactory.stream1.descriptor) + val queue2 = recordQueueSupplier.get(MockDestinationCatalogFactory.stream2.descriptor) - val manager1 = syncManager.getStreamManager(stream1.descriptor) - val manager2 = syncManager.getStreamManager(stream2.descriptor) + val manager1 = + syncManager.getStreamManager(MockDestinationCatalogFactory.stream1.descriptor) + val manager2 = + syncManager.getStreamManager(MockDestinationCatalogFactory.stream2.descriptor) - (0 until 10).forEach { _ -> mockInputFlow.addMessage(makeRecord(stream1, "whatever"), 0L) } + (0 until 10).forEach { _ -> + mockInputFlow.addMessage( + makeRecord(MockDestinationCatalogFactory.stream1, "whatever"), + 0L + ) + } - mockInputFlow.addMessage(makeRecord(stream2, "test"), 1L) - mockInputFlow.addMessage(makeStreamComplete(stream1), 0L) + mockInputFlow.addMessage(makeRecord(MockDestinationCatalogFactory.stream2, "test"), 1L) + mockInputFlow.addMessage(makeStreamComplete(MockDestinationCatalogFactory.stream1), 0L) val job = launch { task.execute() } mockInputFlow.stop() job.join() queue2.close() Assertions.assertEquals( - listOf(StreamRecordWrapped(0, 1L, makeRecord(stream2, "test"))), + listOf( + StreamRecordWrapped( + 0, + 1L, + makeRecord(MockDestinationCatalogFactory.stream2, "test") + ) + ), queue2.consume().toList().map { it.value } ) Assertions.assertEquals(1L, manager2.recordCount()) @@ -209,10 +232,10 @@ class InputConsumerTaskTest { val batches = listOf( - TestEvent(stream1, 10, 10), - TestEvent(stream1, 5, 15), - TestEvent(stream2, 4, 4), - TestEvent(stream1, 3, 18), + TestEvent(MockDestinationCatalogFactory.stream1, 10, 10), + TestEvent(MockDestinationCatalogFactory.stream1, 5, 15), + TestEvent(MockDestinationCatalogFactory.stream2, 4, 4), + TestEvent(MockDestinationCatalogFactory.stream1, 3, 18), ) launch { task.execute() } @@ -239,12 +262,12 @@ class InputConsumerTaskTest { val batches = listOf( - AddRecords(stream1, 10), + AddRecords(MockDestinationCatalogFactory.stream1, 10), SendState(10, 0, 10), - AddRecords(stream2, 5), - AddRecords(stream1, 4), + AddRecords(MockDestinationCatalogFactory.stream2, 5), + AddRecords(MockDestinationCatalogFactory.stream1, 4), SendState(14, 5, 9), - AddRecords(stream2, 3), + AddRecords(MockDestinationCatalogFactory.stream2, 3), SendState(14, 8, 3), SendState(14, 8, 0), ) @@ -262,8 +285,14 @@ class InputConsumerTaskTest { val state = checkpointQueue.consume().take(1).toList().first().value as GlobalCheckpointWrapped - val stream1State = state.streamIndexes.find { it.first == stream1.descriptor }!! - val stream2State = state.streamIndexes.find { it.first == stream2.descriptor }!! + val stream1State = + state.streamIndexes.find { + it.first == MockDestinationCatalogFactory.stream1.descriptor + }!! + val stream2State = + state.streamIndexes.find { + it.first == MockDestinationCatalogFactory.stream2.descriptor + }!! Assertions.assertEquals(event.expectedStream1Count, stream1State.second) Assertions.assertEquals(event.expectedStream2Count, stream2State.second) Assertions.assertEquals( @@ -278,8 +307,8 @@ class InputConsumerTaskTest { @Test fun testStreamIncompleteThrows() = runTest { - mockInputFlow.addMessage(makeRecord(stream1, "test"), 1L) - mockInputFlow.addMessage(makeStreamIncomplete(stream1), 0L) + mockInputFlow.addMessage(makeRecord(MockDestinationCatalogFactory.stream1, "test"), 1L) + mockInputFlow.addMessage(makeStreamIncomplete(MockDestinationCatalogFactory.stream1), 0L) CoroutineTestUtils.assertThrows(IllegalStateException::class) { task.execute() } mockInputFlow.stop() } diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/internal/ReservingDeserializingInputFlowTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/ReservingDeserializingInputFlowTest.kt similarity index 94% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/internal/ReservingDeserializingInputFlowTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/ReservingDeserializingInputFlowTest.kt index 7702ba818e4..91c2faf8b6d 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/internal/ReservingDeserializingInputFlowTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/ReservingDeserializingInputFlowTest.kt @@ -2,11 +2,11 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.internal +package io.airbyte.cdk.load.task.internal -import io.airbyte.cdk.command.DestinationConfiguration -import io.airbyte.cdk.message.Deserializer -import io.airbyte.cdk.state.MemoryManager +import io.airbyte.cdk.load.command.DestinationConfiguration +import io.airbyte.cdk.load.message.Deserializer +import io.airbyte.cdk.load.state.MemoryManager import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires import io.micronaut.test.extensions.junit5.annotation.MicronautTest diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/internal/SpillToDiskTaskTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTaskTest.kt similarity index 77% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/internal/SpillToDiskTaskTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTaskTest.kt index 97df060f331..383e345f1ad 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/internal/SpillToDiskTaskTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTaskTest.kt @@ -2,22 +2,22 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.internal +package io.airbyte.cdk.load.task.internal import com.google.common.collect.Range -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1 -import io.airbyte.cdk.data.NullValue -import io.airbyte.cdk.file.MockTempFileProvider -import io.airbyte.cdk.message.DestinationRecord -import io.airbyte.cdk.message.DestinationRecordWrapped -import io.airbyte.cdk.message.MessageQueueSupplier -import io.airbyte.cdk.message.StreamCompleteWrapped -import io.airbyte.cdk.message.StreamRecordWrapped -import io.airbyte.cdk.state.FlushStrategy -import io.airbyte.cdk.state.MemoryManager -import io.airbyte.cdk.state.Reserved -import io.airbyte.cdk.task.MockTaskLauncher +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.command.MockDestinationCatalogFactory +import io.airbyte.cdk.load.data.NullValue +import io.airbyte.cdk.load.file.MockTempFileProvider +import io.airbyte.cdk.load.message.DestinationRecord +import io.airbyte.cdk.load.message.DestinationRecordWrapped +import io.airbyte.cdk.load.message.MessageQueueSupplier +import io.airbyte.cdk.load.message.StreamCompleteWrapped +import io.airbyte.cdk.load.message.StreamRecordWrapped +import io.airbyte.cdk.load.state.FlushStrategy +import io.airbyte.cdk.load.state.MemoryManager +import io.airbyte.cdk.load.state.Reserved +import io.airbyte.cdk.load.task.MockTaskLauncher import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires import io.micronaut.test.extensions.junit5.annotation.MicronautTest @@ -59,7 +59,7 @@ class SpillToDiskTaskTest { } private suspend fun primeMessageQueue(): Long { - val queue = queueSupplier.get(stream1.descriptor) + val queue = queueSupplier.get(MockDestinationCatalogFactory.stream1.descriptor) val maxRecords = ((1024 * 1.5) / 8).toLong() var recordsWritten = 0L var bytesReserved = 0L @@ -74,7 +74,7 @@ class SpillToDiskTaskTest { sizeBytes = 8, record = DestinationRecord( - stream = stream1.descriptor, + stream = MockDestinationCatalogFactory.stream1.descriptor, data = NullValue, emittedAtMs = 0, meta = null, @@ -95,9 +95,13 @@ class SpillToDiskTaskTest { Assertions.assertEquals(availableMemory - bytesReserved, memoryManager.remainingMemoryBytes) val mockTaskLauncher = MockTaskLauncher() - spillToDiskTaskFactory.make(mockTaskLauncher, stream1).execute() + spillToDiskTaskFactory + .make(mockTaskLauncher, MockDestinationCatalogFactory.stream1) + .execute() Assertions.assertEquals(1, mockTaskLauncher.spilledFiles.size) - spillToDiskTaskFactory.make(mockTaskLauncher, stream1).execute() + spillToDiskTaskFactory + .make(mockTaskLauncher, MockDestinationCatalogFactory.stream1) + .execute() Assertions.assertEquals(2, mockTaskLauncher.spilledFiles.size) Assertions.assertEquals(1024, mockTaskLauncher.spilledFiles[0].batch.totalSizeBytes) diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/internal/TimedForcedCheckpointFlushTaskTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/TimedForcedCheckpointFlushTaskTest.kt similarity index 89% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/internal/TimedForcedCheckpointFlushTaskTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/TimedForcedCheckpointFlushTaskTest.kt index c70421a57af..fc2607f0e77 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/task/internal/TimedForcedCheckpointFlushTaskTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/TimedForcedCheckpointFlushTaskTest.kt @@ -2,14 +2,14 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.task.internal +package io.airbyte.cdk.load.task.internal -import io.airbyte.cdk.command.DestinationConfiguration -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.file.MockTimeProvider -import io.airbyte.cdk.message.QueueReader -import io.airbyte.cdk.state.MockCheckpointManager -import io.airbyte.cdk.task.MockTaskLauncher +import io.airbyte.cdk.load.command.DestinationConfiguration +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.file.MockTimeProvider +import io.airbyte.cdk.load.message.QueueReader +import io.airbyte.cdk.load.state.MockCheckpointManager +import io.airbyte.cdk.load.task.MockTaskLauncher import io.micronaut.test.extensions.junit5.annotation.MicronautTest import jakarta.inject.Inject import kotlinx.coroutines.test.runTest diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/test/util/CoroutineTestUtils.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/test/util/CoroutineTestUtils.kt similarity index 95% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/test/util/CoroutineTestUtils.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/test/util/CoroutineTestUtils.kt index 18e654caf03..a63cfed696b 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/test/util/CoroutineTestUtils.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/test/util/CoroutineTestUtils.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.util +package io.airbyte.cdk.load.test.util import kotlin.reflect.KClass diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/test/util/RecordDifferTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/test/util/RecordDifferTest.kt similarity index 99% rename from airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/test/util/RecordDifferTest.kt rename to airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/test/util/RecordDifferTest.kt index 2c1183dfa0d..030d761a35e 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/test/util/RecordDifferTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/test/util/RecordDifferTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.util +package io.airbyte.cdk.load.test.util import java.time.OffsetDateTime import org.junit.jupiter.api.Assertions diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/check/CheckIntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/check/CheckIntegrationTest.kt similarity index 91% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/check/CheckIntegrationTest.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/check/CheckIntegrationTest.kt index 91ac05ebe34..f7eb2539205 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/check/CheckIntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/check/CheckIntegrationTest.kt @@ -2,15 +2,15 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.check +package io.airbyte.cdk.load.check import io.airbyte.cdk.command.ConfigurationSpecification import io.airbyte.cdk.command.ValidatedJsonUtils -import io.airbyte.cdk.test.util.FakeDataDumper -import io.airbyte.cdk.test.util.IntegrationTest -import io.airbyte.cdk.test.util.NoopDestinationCleaner -import io.airbyte.cdk.test.util.NoopExpectedRecordMapper -import io.airbyte.cdk.test.util.destination_process.TestDeploymentMode +import io.airbyte.cdk.load.test.util.FakeDataDumper +import io.airbyte.cdk.load.test.util.IntegrationTest +import io.airbyte.cdk.load.test.util.NoopDestinationCleaner +import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper +import io.airbyte.cdk.load.test.util.destination_process.TestDeploymentMode import io.airbyte.protocol.models.v0.AirbyteConnectionStatus import io.airbyte.protocol.models.v0.AirbyteMessage import java.nio.charset.StandardCharsets diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/spec/SpecTest.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/spec/SpecTest.kt similarity index 83% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/spec/SpecTest.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/spec/SpecTest.kt index f1d1340f2f2..a3c14cee1ad 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/spec/SpecTest.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/spec/SpecTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.spec +package io.airbyte.cdk.load.spec import com.deblock.jsondiff.DiffGenerator import com.deblock.jsondiff.diff.JsonDiff @@ -12,17 +12,16 @@ import com.deblock.jsondiff.matcher.LenientJsonObjectPartialMatcher import com.deblock.jsondiff.matcher.StrictJsonArrayPartialMatcher import com.deblock.jsondiff.matcher.StrictPrimitivePartialMatcher import com.deblock.jsondiff.viewer.OnlyErrorDiffViewer -import io.airbyte.cdk.test.util.FakeDataDumper -import io.airbyte.cdk.test.util.IntegrationTest -import io.airbyte.cdk.test.util.NoopDestinationCleaner -import io.airbyte.cdk.test.util.NoopExpectedRecordMapper -import io.airbyte.cdk.test.util.destination_process.DestinationProcessFactory -import io.airbyte.cdk.test.util.destination_process.TestDeploymentMode +import io.airbyte.cdk.load.test.util.FakeDataDumper +import io.airbyte.cdk.load.test.util.IntegrationTest +import io.airbyte.cdk.load.test.util.NoopDestinationCleaner +import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper +import io.airbyte.cdk.load.test.util.destination_process.TestDeploymentMode import io.airbyte.cdk.util.Jsons import io.airbyte.protocol.models.v0.AirbyteMessage import java.nio.file.Files import java.nio.file.Path -import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertAll @@ -67,7 +66,7 @@ abstract class SpecTest : val messages = process.readMessages() val specMessages = messages.filter { it.type == AirbyteMessage.Type.SPEC } - assertEquals( + Assertions.assertEquals( specMessages.size, 1, "Expected to receive exactly one connection status message, but got ${specMessages.size}: $specMessages" @@ -88,8 +87,8 @@ abstract class SpecTest : DiffGenerator.diff(expectedSpec, Jsons.writeValueAsString(spec), jsonMatcher) assertAll( "Spec snapshot test failed. Run this test locally and then `git diff <...>/$expectedSpecFilename` to see what changed, and commit the diff if that change was intentional.", - { assertEquals("", OnlyErrorDiffViewer.from(diff).toString()) }, - { assertEquals(expectedSpec, actualSpecPrettyPrint) } + { Assertions.assertEquals("", OnlyErrorDiffViewer.from(diff).toString()) }, + { Assertions.assertEquals(expectedSpec, actualSpecPrettyPrint) } ) } } diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/DestinationCleaner.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/DestinationCleaner.kt similarity index 91% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/DestinationCleaner.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/DestinationCleaner.kt index a30c90d90a0..1ab6c2e20f1 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/DestinationCleaner.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/DestinationCleaner.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.util +package io.airbyte.cdk.load.test.util fun interface DestinationCleaner { /** diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/DestinationDataDumper.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/DestinationDataDumper.kt similarity index 93% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/DestinationDataDumper.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/DestinationDataDumper.kt index c50ddf02bc3..93d68879df9 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/DestinationDataDumper.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/DestinationDataDumper.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.util +package io.airbyte.cdk.load.test.util fun interface DestinationDataDumper { fun dumpRecords( diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/ExpectedRecordMapper.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/ExpectedRecordMapper.kt similarity index 89% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/ExpectedRecordMapper.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/ExpectedRecordMapper.kt index 22746f6629c..d500e8398b9 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/ExpectedRecordMapper.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/ExpectedRecordMapper.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.util +package io.airbyte.cdk.load.test.util fun interface ExpectedRecordMapper { fun mapRecord(expectedRecord: OutputRecord): OutputRecord diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/IntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt similarity index 95% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/IntegrationTest.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt index 7eed61d8bd6..a0fae6a3696 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/IntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt @@ -2,14 +2,14 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.util +package io.airbyte.cdk.load.test.util import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import io.airbyte.cdk.command.ConfigurationSpecification -import io.airbyte.cdk.command.DestinationCatalog -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.message.DestinationMessage -import io.airbyte.cdk.test.util.destination_process.DestinationProcessFactory +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.message.DestinationMessage +import io.airbyte.cdk.load.test.util.destination_process.DestinationProcessFactory import io.airbyte.protocol.models.v0.AirbyteMessage import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage.AirbyteStreamStatus diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/NameMapper.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/NameMapper.kt similarity index 94% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/NameMapper.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/NameMapper.kt index 9ff5a38d791..3d87bb66179 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/NameMapper.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/NameMapper.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.util +package io.airbyte.cdk.load.test.util fun interface NameMapper { /** diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/OutputRecord.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/OutputRecord.kt similarity index 93% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/OutputRecord.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/OutputRecord.kt index 18ac7545d50..eea70838e18 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/OutputRecord.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/OutputRecord.kt @@ -2,10 +2,10 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.util +package io.airbyte.cdk.load.test.util -import io.airbyte.cdk.data.ObjectValue -import io.airbyte.cdk.message.DestinationRecord.Change +import io.airbyte.cdk.load.data.ObjectValue +import io.airbyte.cdk.load.message.DestinationRecord.Change import java.time.Instant import java.util.UUID diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/RecordDiffer.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/RecordDiffer.kt similarity index 98% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/RecordDiffer.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/RecordDiffer.kt index 5c18d3954e2..ce1b7026253 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/RecordDiffer.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/RecordDiffer.kt @@ -2,12 +2,12 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.util +package io.airbyte.cdk.load.test.util -import io.airbyte.cdk.data.AirbyteValue -import io.airbyte.cdk.data.IntegerValue -import io.airbyte.cdk.data.NullValue -import io.airbyte.cdk.data.ObjectValue +import io.airbyte.cdk.load.data.AirbyteValue +import io.airbyte.cdk.load.data.IntegerValue +import io.airbyte.cdk.load.data.NullValue +import io.airbyte.cdk.load.data.ObjectValue import kotlin.reflect.jvm.jvmName class RecordDiffer( diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/destination_process/DestinationProcess.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt similarity index 96% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/destination_process/DestinationProcess.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt index ce8aab8fc15..7b0b8a26bd3 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/destination_process/DestinationProcess.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.util.destination_process +package io.airbyte.cdk.load.test.util.destination_process import io.airbyte.cdk.command.ConfigurationSpecification import io.airbyte.protocol.models.v0.AirbyteMessage diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/destination_process/DockerizedDestination.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt similarity index 97% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/destination_process/DockerizedDestination.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt index b0886138e51..ab09197108e 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/destination_process/DockerizedDestination.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.util.destination_process +package io.airbyte.cdk.load.test.util.destination_process import com.fasterxml.jackson.databind.JsonNode import io.airbyte.protocol.models.v0.AirbyteMessage diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/destination_process/NonDockerizedDestination.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt similarity index 98% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/destination_process/NonDockerizedDestination.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt index ab301f71215..672b2b5911f 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/util/destination_process/NonDockerizedDestination.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.util.destination_process +package io.airbyte.cdk.load.test.util.destination_process import io.airbyte.cdk.command.CliRunnable import io.airbyte.cdk.command.CliRunner diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/write/BasicFunctionalityIntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt similarity index 76% rename from airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/write/BasicFunctionalityIntegrationTest.kt rename to airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt index 7cd18ae8bda..e5b51aee5ae 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/test/write/BasicFunctionalityIntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt @@ -2,26 +2,25 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.test.write +package io.airbyte.cdk.load.write -import io.airbyte.cdk.command.Append import io.airbyte.cdk.command.ConfigurationSpecification -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.data.ObjectTypeWithoutSchema -import io.airbyte.cdk.message.DestinationRecord -import io.airbyte.cdk.message.StreamCheckpoint -import io.airbyte.cdk.test.util.DestinationCleaner -import io.airbyte.cdk.test.util.DestinationDataDumper -import io.airbyte.cdk.test.util.ExpectedRecordMapper -import io.airbyte.cdk.test.util.IntegrationTest -import io.airbyte.cdk.test.util.NameMapper -import io.airbyte.cdk.test.util.NoopExpectedRecordMapper -import io.airbyte.cdk.test.util.NoopNameMapper -import io.airbyte.cdk.test.util.OutputRecord +import io.airbyte.cdk.load.command.Append +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.data.ObjectTypeWithoutSchema +import io.airbyte.cdk.load.message.DestinationRecord +import io.airbyte.cdk.load.message.StreamCheckpoint +import io.airbyte.cdk.load.test.util.DestinationCleaner +import io.airbyte.cdk.load.test.util.DestinationDataDumper +import io.airbyte.cdk.load.test.util.ExpectedRecordMapper +import io.airbyte.cdk.load.test.util.IntegrationTest +import io.airbyte.cdk.load.test.util.NameMapper +import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper +import io.airbyte.cdk.load.test.util.NoopNameMapper +import io.airbyte.cdk.load.test.util.OutputRecord import io.airbyte.protocol.models.v0.AirbyteConnectionStatus import io.airbyte.protocol.models.v0.AirbyteMessage -import io.airbyte.protocol.models.v0.AirbyteRecordMessageMetaChange.Change -import io.airbyte.protocol.models.v0.AirbyteRecordMessageMetaChange.Reason +import io.airbyte.protocol.models.v0.AirbyteRecordMessageMetaChange import kotlin.test.assertEquals import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertAll @@ -79,8 +78,10 @@ abstract class BasicFunctionalityIntegrationTest( listOf( DestinationRecord.Change( field = "foo", - change = Change.NULLED, - reason = Reason.SOURCE_FIELD_SIZE_LIMITATION + change = AirbyteRecordMessageMetaChange.Change.NULLED, + reason = + AirbyteRecordMessageMetaChange.Reason + .SOURCE_FIELD_SIZE_LIMITATION ) ) ), @@ -127,8 +128,12 @@ abstract class BasicFunctionalityIntegrationTest( listOf( DestinationRecord.Change( field = "foo", - change = Change.NULLED, - reason = Reason.SOURCE_FIELD_SIZE_LIMITATION + change = + AirbyteRecordMessageMetaChange.Change + .NULLED, + reason = + AirbyteRecordMessageMetaChange.Reason + .SOURCE_FIELD_SIZE_LIMITATION ) ), syncId = 42 diff --git a/airbyte-cdk/bulk/toolkits/extract-jdbc/src/main/kotlin/io/airbyte/cdk/discover/JdbcAirbyteStreamFactory.kt b/airbyte-cdk/bulk/toolkits/extract-jdbc/src/main/kotlin/io/airbyte/cdk/discover/JdbcAirbyteStreamFactory.kt index 8818a038077..5a19f8aac9c 100644 --- a/airbyte-cdk/bulk/toolkits/extract-jdbc/src/main/kotlin/io/airbyte/cdk/discover/JdbcAirbyteStreamFactory.kt +++ b/airbyte-cdk/bulk/toolkits/extract-jdbc/src/main/kotlin/io/airbyte/cdk/discover/JdbcAirbyteStreamFactory.kt @@ -20,7 +20,7 @@ class JdbcAirbyteStreamFactory : AirbyteStreamFactory { supportedSyncModes = listOf(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL) (jsonSchema["properties"] as ObjectNode).apply { for (metaField in CommonMetaField.entries) { - set(metaField.id, metaField.type.airbyteType.asJsonSchema()) + set(metaField.id, metaField.type.airbyteSchemaType.asJsonSchema()) } } defaultCursorField = listOf(CommonMetaField.CDC_LSN.id) diff --git a/airbyte-cdk/bulk/toolkits/extract-jdbc/src/main/kotlin/io/airbyte/cdk/jdbc/JdbcFieldTypes.kt b/airbyte-cdk/bulk/toolkits/extract-jdbc/src/main/kotlin/io/airbyte/cdk/jdbc/JdbcFieldTypes.kt index 44b0ba036ae..5b193d96de3 100644 --- a/airbyte-cdk/bulk/toolkits/extract-jdbc/src/main/kotlin/io/airbyte/cdk/jdbc/JdbcFieldTypes.kt +++ b/airbyte-cdk/bulk/toolkits/extract-jdbc/src/main/kotlin/io/airbyte/cdk/jdbc/JdbcFieldTypes.kt @@ -2,9 +2,9 @@ package io.airbyte.cdk.jdbc import com.fasterxml.jackson.databind.JsonNode -import io.airbyte.cdk.data.AirbyteType +import io.airbyte.cdk.data.AirbyteSchemaType import io.airbyte.cdk.data.AnyEncoder -import io.airbyte.cdk.data.ArrayAirbyteType +import io.airbyte.cdk.data.ArrayAirbyteSchemaType import io.airbyte.cdk.data.ArrayEncoder import io.airbyte.cdk.data.BigDecimalCodec import io.airbyte.cdk.data.BigDecimalIntegerCodec @@ -18,7 +18,7 @@ import io.airbyte.cdk.data.JsonCodec import io.airbyte.cdk.data.JsonDecoder import io.airbyte.cdk.data.JsonEncoder import io.airbyte.cdk.data.JsonStringCodec -import io.airbyte.cdk.data.LeafAirbyteType +import io.airbyte.cdk.data.LeafAirbyteSchemaType import io.airbyte.cdk.data.LocalDateCodec import io.airbyte.cdk.data.LocalDateTimeCodec import io.airbyte.cdk.data.LocalTimeCodec @@ -44,7 +44,7 @@ import java.time.OffsetTime /** Convenience class for defining concrete [FieldType] objects. */ abstract class JdbcFieldType( - override val airbyteType: AirbyteType, + override val airbyteSchemaType: AirbyteSchemaType, val jdbcGetter: JdbcGetter, override val jsonEncoder: JsonEncoder, ) : FieldType, JdbcGetter { @@ -60,13 +60,13 @@ abstract class JdbcFieldType( /** Convenience class for defining concrete [LosslessFieldType] objects. */ abstract class LosslessJdbcFieldType( - airbyteType: AirbyteType, + airbyteSchemaType: AirbyteSchemaType, jdbcGetter: JdbcGetter, jsonEncoder: JsonEncoder, override val jsonDecoder: JsonDecoder, val jdbcSetter: JdbcSetter, ) : - JdbcFieldType(airbyteType, jdbcGetter, jsonEncoder), + JdbcFieldType(airbyteSchemaType, jdbcGetter, jsonEncoder), LosslessFieldType, JdbcSetter { override fun set( @@ -80,175 +80,182 @@ abstract class LosslessJdbcFieldType( /** Convenience class for defining concrete [LosslessFieldType] objects. */ abstract class SymmetricJdbcFieldType( - airbyteType: AirbyteType, + airbyteSchemaType: AirbyteSchemaType, jdbcAccessor: JdbcAccessor, jsonCodec: JsonCodec, -) : LosslessJdbcFieldType(airbyteType, jdbcAccessor, jsonCodec, jsonCodec, jdbcAccessor) +) : + LosslessJdbcFieldType( + airbyteSchemaType, + jdbcAccessor, + jsonCodec, + jsonCodec, + jdbcAccessor + ) data object BooleanFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.BOOLEAN, + LeafAirbyteSchemaType.BOOLEAN, BooleanAccessor, BooleanCodec, ) data object BigDecimalFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.NUMBER, + LeafAirbyteSchemaType.NUMBER, BigDecimalAccessor, BigDecimalCodec, ) data object DoubleFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.NUMBER, + LeafAirbyteSchemaType.NUMBER, DoubleAccessor, DoubleCodec, ) data object FloatFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.NUMBER, + LeafAirbyteSchemaType.NUMBER, FloatAccessor, FloatCodec, ) data object BigIntegerFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.INTEGER, + LeafAirbyteSchemaType.INTEGER, BigDecimalAccessor, BigDecimalIntegerCodec, ) data object LongFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.INTEGER, + LeafAirbyteSchemaType.INTEGER, LongAccessor, LongCodec, ) data object IntFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.INTEGER, + LeafAirbyteSchemaType.INTEGER, IntAccessor, IntCodec, ) data object ShortFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.INTEGER, + LeafAirbyteSchemaType.INTEGER, ShortAccessor, ShortCodec, ) data object ByteFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.INTEGER, + LeafAirbyteSchemaType.INTEGER, ByteAccessor, ByteCodec, ) data object StringFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.STRING, + LeafAirbyteSchemaType.STRING, StringAccessor, TextCodec, ) data object NStringFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.STRING, + LeafAirbyteSchemaType.STRING, NStringAccessor, TextCodec, ) data object CharacterStreamFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.STRING, + LeafAirbyteSchemaType.STRING, CharacterStreamAccessor, TextCodec, ) data object NCharacterStreamFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.STRING, + LeafAirbyteSchemaType.STRING, NCharacterStreamAccessor, TextCodec, ) data object ClobFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.STRING, + LeafAirbyteSchemaType.STRING, ClobAccessor, TextCodec, ) data object NClobFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.STRING, + LeafAirbyteSchemaType.STRING, NClobAccessor, TextCodec, ) data object XmlFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.STRING, + LeafAirbyteSchemaType.STRING, XmlAccessor, TextCodec, ) data object UrlFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.STRING, + LeafAirbyteSchemaType.STRING, UrlAccessor, UrlCodec, ) data object BytesFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.BINARY, + LeafAirbyteSchemaType.BINARY, BytesAccessor, BinaryCodec, ) data object BinaryStreamFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.BINARY, + LeafAirbyteSchemaType.BINARY, BinaryStreamAccessor, BinaryCodec, ) data object JsonStringFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.JSONB, + LeafAirbyteSchemaType.JSONB, StringAccessor, JsonStringCodec, ) data object LocalDateFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.DATE, + LeafAirbyteSchemaType.DATE, DateAccessor, LocalDateCodec, ) data object LocalTimeFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.TIME_WITHOUT_TIMEZONE, + LeafAirbyteSchemaType.TIME_WITHOUT_TIMEZONE, TimeAccessor, LocalTimeCodec, ) data object LocalDateTimeFieldType : SymmetricJdbcFieldType( - LeafAirbyteType.TIMESTAMP_WITHOUT_TIMEZONE, + LeafAirbyteSchemaType.TIMESTAMP_WITHOUT_TIMEZONE, TimestampAccessor, LocalDateTimeCodec, ) data object OffsetTimeFieldType : LosslessJdbcFieldType( - LeafAirbyteType.TIME_WITH_TIMEZONE, + LeafAirbyteSchemaType.TIME_WITH_TIMEZONE, ObjectGetter(OffsetTime::class.java), OffsetTimeCodec, OffsetTimeCodec, @@ -257,7 +264,7 @@ data object OffsetTimeFieldType : data object OffsetDateTimeFieldType : LosslessJdbcFieldType( - LeafAirbyteType.TIMESTAMP_WITH_TIMEZONE, + LeafAirbyteSchemaType.TIMESTAMP_WITH_TIMEZONE, ObjectGetter(OffsetDateTime::class.java), OffsetDateTimeCodec, OffsetDateTimeCodec, @@ -266,14 +273,14 @@ data object OffsetDateTimeFieldType : data object PokemonFieldType : JdbcFieldType( - LeafAirbyteType.STRING, + LeafAirbyteSchemaType.STRING, StringAccessor, AnyEncoder, ) data object NullFieldType : JdbcFieldType( - LeafAirbyteType.NULL, + LeafAirbyteSchemaType.NULL, AnyAccessor, NullCodec, ) @@ -282,7 +289,7 @@ data class ArrayFieldType( val elementFieldType: JdbcFieldType, ) : JdbcFieldType>( - ArrayAirbyteType(elementFieldType.airbyteType), + ArrayAirbyteSchemaType(elementFieldType.airbyteSchemaType), ArrayGetter(elementFieldType.jdbcGetter), ArrayEncoder(elementFieldType.jsonEncoder), ) diff --git a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml index cc57ffecbd4..fc32ebeb425 100644 --- a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml +++ b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: a7bcc9d8-13b3-4e49-b80d-d020b90045e3 - dockerImageTag: 0.7.3 + dockerImageTag: 0.7.4 dockerRepository: airbyte/destination-dev-null githubIssueLabel: destination-dev-null icon: airbyte.svg diff --git a/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullChecker.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullChecker.kt index 319dbadb9c2..dcbd95bc8ec 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullChecker.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullChecker.kt @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.dev_null -import io.airbyte.cdk.check.DestinationChecker +import io.airbyte.cdk.load.check.DestinationChecker import jakarta.inject.Singleton @Singleton diff --git a/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullConfiguration.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullConfiguration.kt index 5e868eb1e93..6de60f3dd8b 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullConfiguration.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullConfiguration.kt @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.dev_null -import io.airbyte.cdk.command.DestinationConfiguration -import io.airbyte.cdk.command.DestinationConfigurationFactory +import io.airbyte.cdk.load.command.DestinationConfiguration +import io.airbyte.cdk.load.command.DestinationConfigurationFactory import io.micronaut.context.annotation.Factory import jakarta.inject.Singleton diff --git a/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullSpecification.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullSpecification.kt index f258855f68c..0eb567f8417 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullSpecification.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullSpecification.kt @@ -13,7 +13,7 @@ import com.fasterxml.jackson.annotation.JsonValue import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import io.airbyte.cdk.command.ConfigurationSpecification -import io.airbyte.cdk.spec.DestinationSpecificationExtension +import io.airbyte.cdk.load.spec.DestinationSpecificationExtension import io.airbyte.protocol.models.v0.DestinationSyncMode import io.micronaut.context.annotation.Requires import jakarta.inject.Singleton diff --git a/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullWriter.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullWriter.kt index eed85d211dc..20ec5486e4e 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullWriter.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullWriter.kt @@ -5,12 +5,12 @@ package io.airbyte.integrations.destination.dev_null import edu.umd.cs.findbugs.annotations.SuppressFBWarnings -import io.airbyte.cdk.command.DestinationStream -import io.airbyte.cdk.message.Batch -import io.airbyte.cdk.message.DestinationRecord -import io.airbyte.cdk.message.SimpleBatch -import io.airbyte.cdk.write.DestinationWriter -import io.airbyte.cdk.write.StreamLoader +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.message.Batch +import io.airbyte.cdk.load.message.DestinationRecord +import io.airbyte.cdk.load.message.SimpleBatch +import io.airbyte.cdk.load.write.DestinationWriter +import io.airbyte.cdk.load.write.StreamLoader import io.github.oshai.kotlinlogging.KotlinLogging import jakarta.inject.Singleton import java.util.concurrent.atomic.AtomicInteger diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullBasicFunctionalityIntegrationTest.kt b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullBasicFunctionalityIntegrationTest.kt index 4d7bd503596..c0ecf552744 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullBasicFunctionalityIntegrationTest.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullBasicFunctionalityIntegrationTest.kt @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.dev_null -import io.airbyte.cdk.test.util.NoopDestinationCleaner -import io.airbyte.cdk.test.util.NoopExpectedRecordMapper -import io.airbyte.cdk.test.write.BasicFunctionalityIntegrationTest +import io.airbyte.cdk.load.test.util.NoopDestinationCleaner +import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper +import io.airbyte.cdk.load.write.BasicFunctionalityIntegrationTest import org.junit.jupiter.api.Test class DevNullBasicFunctionalityIntegrationTest : diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullCheckIntegrationTest.kt b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullCheckIntegrationTest.kt index 798eade8eac..5d3539ad251 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullCheckIntegrationTest.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullCheckIntegrationTest.kt @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.dev_null -import io.airbyte.cdk.test.check.CheckIntegrationTest -import io.airbyte.cdk.test.check.CheckTestConfig -import io.airbyte.cdk.test.util.destination_process.TestDeploymentMode +import io.airbyte.cdk.load.check.CheckIntegrationTest +import io.airbyte.cdk.load.check.CheckTestConfig +import io.airbyte.cdk.load.test.util.destination_process.TestDeploymentMode import java.util.regex.Pattern import org.junit.jupiter.api.Test diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationDataDumper.kt b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationDataDumper.kt index f9681796aef..c322c5e26e5 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationDataDumper.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationDataDumper.kt @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.dev_null -import io.airbyte.cdk.test.util.DestinationDataDumper -import io.airbyte.cdk.test.util.OutputRecord +import io.airbyte.cdk.load.test.util.DestinationDataDumper +import io.airbyte.cdk.load.test.util.OutputRecord object DevNullDestinationDataDumper : DestinationDataDumper { override fun dumpRecords(streamName: String, streamNamespace: String?): List { diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullSpecTest.kt b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullSpecTest.kt index 0c80d835a46..8af3c5f9011 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullSpecTest.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullSpecTest.kt @@ -4,6 +4,6 @@ package io.airbyte.integrations.destination.dev_null -import io.airbyte.cdk.test.spec.SpecTest +import io.airbyte.cdk.load.spec.SpecTest class DevNullSpecTest : SpecTest() diff --git a/airbyte-integrations/connectors/source-mysql-v2/metadata.yaml b/airbyte-integrations/connectors/source-mysql-v2/metadata.yaml index baa1cd155d0..f46c784501d 100644 --- a/airbyte-integrations/connectors/source-mysql-v2/metadata.yaml +++ b/airbyte-integrations/connectors/source-mysql-v2/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: 561393ed-7e3a-4d0d-8b8b-90ded371754c - dockerImageTag: 0.0.18 + dockerImageTag: 0.0.19 dockerRepository: airbyte/source-mysql-v2 documentationUrl: https://docs.airbyte.com/integrations/sources/mysql githubIssueLabel: source-mysql-v2 diff --git a/airbyte-integrations/connectors/source-mysql-v2/src/main/kotlin/io/airbyte/integrations/source/mysql/MysqlJdbcStreamFactory.kt b/airbyte-integrations/connectors/source-mysql-v2/src/main/kotlin/io/airbyte/integrations/source/mysql/MysqlJdbcStreamFactory.kt index 9060572a93a..e04c36e6868 100644 --- a/airbyte-integrations/connectors/source-mysql-v2/src/main/kotlin/io/airbyte/integrations/source/mysql/MysqlJdbcStreamFactory.kt +++ b/airbyte-integrations/connectors/source-mysql-v2/src/main/kotlin/io/airbyte/integrations/source/mysql/MysqlJdbcStreamFactory.kt @@ -26,7 +26,7 @@ class MysqlJdbcStreamFactory(val base: JdbcAirbyteStreamFactory) : AirbyteStream return AirbyteStreamFactory.createAirbyteStream(discoveredStream).apply { (jsonSchema["properties"] as ObjectNode).apply { for (metaField in MysqlCDCMetaFields.entries) { - set(metaField.id, metaField.type.airbyteType.asJsonSchema()) + set(metaField.id, metaField.type.airbyteSchemaType.asJsonSchema()) } } if (base.hasValidPrimaryKey(discoveredStream)) { diff --git a/airbyte-integrations/connectors/source-mysql-v2/src/test/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceDatatypeIntegrationTest.kt b/airbyte-integrations/connectors/source-mysql-v2/src/test/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceDatatypeIntegrationTest.kt index 12d4a4ecbcc..5b2eee35bb6 100644 --- a/airbyte-integrations/connectors/source-mysql-v2/src/test/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceDatatypeIntegrationTest.kt +++ b/airbyte-integrations/connectors/source-mysql-v2/src/test/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceDatatypeIntegrationTest.kt @@ -4,8 +4,8 @@ package io.airbyte.integrations.source.mysql import com.fasterxml.jackson.databind.JsonNode import io.airbyte.cdk.ClockFactory import io.airbyte.cdk.command.CliRunner -import io.airbyte.cdk.data.AirbyteType -import io.airbyte.cdk.data.LeafAirbyteType +import io.airbyte.cdk.data.AirbyteSchemaType +import io.airbyte.cdk.data.LeafAirbyteSchemaType import io.airbyte.cdk.jdbc.JdbcConnectionFactory import io.airbyte.cdk.output.BufferingOutputConsumer import io.airbyte.cdk.util.Jsons @@ -127,7 +127,7 @@ class MysqlSourceDatatypeIntegrationTest { if (streamName == testCase.tableName) { val actualSchema: JsonNode = jsonSchema[testCase.columnName] Assertions.assertNotNull(actualSchema) - val expectedSchema: JsonNode = testCase.airbyteType.asJsonSchema() + val expectedSchema: JsonNode = testCase.airbyteSchemaType.asJsonSchema() Assertions.assertEquals(expectedSchema, actualSchema) if (testCase.cursor) { Assertions.assertTrue(isIncrementalSupported) @@ -260,67 +260,133 @@ class MysqlSourceDatatypeIntegrationTest { TestCase( "BOOLEAN", booleanValues, - airbyteType = LeafAirbyteType.BOOLEAN, + airbyteSchemaType = LeafAirbyteSchemaType.BOOLEAN, cursor = false ), - TestCase("VARCHAR(10)", stringValues, airbyteType = LeafAirbyteType.STRING), - TestCase("DECIMAL(10,2)", decimalValues, airbyteType = LeafAirbyteType.NUMBER), + TestCase( + "VARCHAR(10)", + stringValues, + airbyteSchemaType = LeafAirbyteSchemaType.STRING + ), + TestCase( + "DECIMAL(10,2)", + decimalValues, + airbyteSchemaType = LeafAirbyteSchemaType.NUMBER + ), TestCase( "DECIMAL(10,2) UNSIGNED", decimalValues, - airbyteType = LeafAirbyteType.NUMBER + airbyteSchemaType = LeafAirbyteSchemaType.NUMBER ), TestCase( "DECIMAL UNSIGNED", zeroPrecisionDecimalValues, - airbyteType = LeafAirbyteType.INTEGER + airbyteSchemaType = LeafAirbyteSchemaType.INTEGER + ), + TestCase("FLOAT", decimalValues, airbyteSchemaType = LeafAirbyteSchemaType.NUMBER), + TestCase( + "FLOAT(7,4)", + decimalValues, + airbyteSchemaType = LeafAirbyteSchemaType.NUMBER + ), + TestCase( + "FLOAT(53,8)", + decimalValues, + airbyteSchemaType = LeafAirbyteSchemaType.NUMBER + ), + TestCase("DOUBLE", decimalValues, airbyteSchemaType = LeafAirbyteSchemaType.NUMBER), + TestCase( + "DOUBLE UNSIGNED", + decimalValues, + airbyteSchemaType = LeafAirbyteSchemaType.NUMBER + ), + TestCase( + "TINYINT", + tinyintValues, + airbyteSchemaType = LeafAirbyteSchemaType.INTEGER + ), + TestCase( + "TINYINT UNSIGNED", + tinyintValues, + airbyteSchemaType = LeafAirbyteSchemaType.INTEGER + ), + TestCase( + "SMALLINT", + tinyintValues, + airbyteSchemaType = LeafAirbyteSchemaType.INTEGER + ), + TestCase( + "MEDIUMINT", + tinyintValues, + airbyteSchemaType = LeafAirbyteSchemaType.INTEGER + ), + TestCase("BIGINT", intValues, airbyteSchemaType = LeafAirbyteSchemaType.INTEGER), + TestCase( + "SMALLINT UNSIGNED", + tinyintValues, + airbyteSchemaType = LeafAirbyteSchemaType.INTEGER ), - TestCase("FLOAT", decimalValues, airbyteType = LeafAirbyteType.NUMBER), - TestCase("FLOAT(7,4)", decimalValues, airbyteType = LeafAirbyteType.NUMBER), - TestCase("FLOAT(53,8)", decimalValues, airbyteType = LeafAirbyteType.NUMBER), - TestCase("DOUBLE", decimalValues, airbyteType = LeafAirbyteType.NUMBER), - TestCase("DOUBLE UNSIGNED", decimalValues, airbyteType = LeafAirbyteType.NUMBER), - TestCase("TINYINT", tinyintValues, airbyteType = LeafAirbyteType.INTEGER), - TestCase("TINYINT UNSIGNED", tinyintValues, airbyteType = LeafAirbyteType.INTEGER), - TestCase("SMALLINT", tinyintValues, airbyteType = LeafAirbyteType.INTEGER), - TestCase("MEDIUMINT", tinyintValues, airbyteType = LeafAirbyteType.INTEGER), - TestCase("BIGINT", intValues, airbyteType = LeafAirbyteType.INTEGER), - TestCase("SMALLINT UNSIGNED", tinyintValues, airbyteType = LeafAirbyteType.INTEGER), TestCase( "MEDIUMINT UNSIGNED", tinyintValues, - airbyteType = LeafAirbyteType.INTEGER + airbyteSchemaType = LeafAirbyteSchemaType.INTEGER ), - TestCase("BIGINT UNSIGNED", intValues, airbyteType = LeafAirbyteType.INTEGER), - TestCase("INT", intValues, airbyteType = LeafAirbyteType.INTEGER), - TestCase("INT UNSIGNED", intValues, airbyteType = LeafAirbyteType.INTEGER), - TestCase("DATE", dateValues, airbyteType = LeafAirbyteType.DATE), + TestCase( + "BIGINT UNSIGNED", + intValues, + airbyteSchemaType = LeafAirbyteSchemaType.INTEGER + ), + TestCase("INT", intValues, airbyteSchemaType = LeafAirbyteSchemaType.INTEGER), + TestCase( + "INT UNSIGNED", + intValues, + airbyteSchemaType = LeafAirbyteSchemaType.INTEGER + ), + TestCase("DATE", dateValues, airbyteSchemaType = LeafAirbyteSchemaType.DATE), TestCase( "TIMESTAMP", timestampValues, - airbyteType = LeafAirbyteType.TIMESTAMP_WITH_TIMEZONE + airbyteSchemaType = LeafAirbyteSchemaType.TIMESTAMP_WITH_TIMEZONE ), TestCase( "DATETIME", dateTimeValues, - airbyteType = LeafAirbyteType.TIMESTAMP_WITHOUT_TIMEZONE + airbyteSchemaType = LeafAirbyteSchemaType.TIMESTAMP_WITHOUT_TIMEZONE ), - TestCase("TIME", timeValues, airbyteType = LeafAirbyteType.TIME_WITHOUT_TIMEZONE), - TestCase("YEAR", yearValues, airbyteType = LeafAirbyteType.INTEGER), + TestCase( + "TIME", + timeValues, + airbyteSchemaType = LeafAirbyteSchemaType.TIME_WITHOUT_TIMEZONE + ), + TestCase("YEAR", yearValues, airbyteSchemaType = LeafAirbyteSchemaType.INTEGER), TestCase( "VARBINARY(255)", binaryValues, - airbyteType = LeafAirbyteType.BINARY, + airbyteSchemaType = LeafAirbyteSchemaType.BINARY, cursor = false, noPK = true ), - TestCase("BIT", bitValues, airbyteType = LeafAirbyteType.BOOLEAN, cursor = false), - TestCase("BIT(8)", longBitValues, airbyteType = LeafAirbyteType.INTEGER), - TestCase("JSON", jsonValues, airbyteType = LeafAirbyteType.STRING, noPK = true), + TestCase( + "BIT", + bitValues, + airbyteSchemaType = LeafAirbyteSchemaType.BOOLEAN, + cursor = false + ), + TestCase( + "BIT(8)", + longBitValues, + airbyteSchemaType = LeafAirbyteSchemaType.INTEGER + ), + TestCase( + "JSON", + jsonValues, + airbyteSchemaType = LeafAirbyteSchemaType.STRING, + noPK = true + ), TestCase( "ENUM('a', 'b', 'c')", enumValues, - airbyteType = LeafAirbyteType.STRING, + airbyteSchemaType = LeafAirbyteSchemaType.STRING, noPK = true ), ) @@ -354,7 +420,7 @@ class MysqlSourceDatatypeIntegrationTest { data class TestCase( val sqlType: String, val sqlToAirbyte: Map, - val airbyteType: AirbyteType = LeafAirbyteType.STRING, + val airbyteSchemaType: AirbyteSchemaType = LeafAirbyteSchemaType.STRING, val cursor: Boolean = true, val noPK: Boolean = false, val customDDL: List? = null, diff --git a/docs/integrations/destinations/dev-null.md b/docs/integrations/destinations/dev-null.md index b65bb1782a7..1b0dcd21d15 100644 --- a/docs/integrations/destinations/dev-null.md +++ b/docs/integrations/destinations/dev-null.md @@ -49,6 +49,7 @@ The OSS and Cloud variants have the same version number starting from version `0 | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------| +| 0.7.4 | 2024-10-08 | [46650](https://github.com/airbytehq/airbyte/pull/46650) | Internal code changes | | 0.7.3 | 2024-10-01 | [46559](https://github.com/airbytehq/airbyte/pull/46559) | From load CDK: async improvements, stream incomplete, additionalProperties on state messages | | 0.7.2 | 2024-10-01 | [45929](https://github.com/airbytehq/airbyte/pull/45929) | Internal code changes | | 0.7.1 | 2024-09-30 | [46276](https://github.com/airbytehq/airbyte/pull/46276) | Upgrade to latest bulk CDK | diff --git a/spotbugs-exclude-filter-file.xml b/spotbugs-exclude-filter-file.xml index 6b825caa6c3..f37159bac3d 100644 --- a/spotbugs-exclude-filter-file.xml +++ b/spotbugs-exclude-filter-file.xml @@ -20,6 +20,6 @@ - +