Bulk CDK: Rename stuff to prevent conflicts (#46650)
This commit is contained in:
@@ -12,7 +12,7 @@ import io.airbyte.protocol.models.JsonSchemaType
|
||||
* This maps to the subset of [JsonSchemaType] which is used in practice. Its main reason for
|
||||
* existing is to provide type-safety and convenient comparisons and string representations.
|
||||
*/
|
||||
sealed interface AirbyteType {
|
||||
sealed interface AirbyteSchemaType {
|
||||
/** Unwraps the underlying Airbyte protocol type object. */
|
||||
fun asJsonSchemaType(): JsonSchemaType
|
||||
|
||||
@@ -20,18 +20,18 @@ sealed interface AirbyteType {
|
||||
fun asJsonSchema(): JsonNode = Jsons.valueToTree(asJsonSchemaType().jsonSchemaTypeMap)
|
||||
}
|
||||
|
||||
data class ArrayAirbyteType(
|
||||
val item: AirbyteType,
|
||||
) : AirbyteType {
|
||||
data class ArrayAirbyteSchemaType(
|
||||
val item: AirbyteSchemaType,
|
||||
) : AirbyteSchemaType {
|
||||
override fun asJsonSchemaType(): JsonSchemaType =
|
||||
JsonSchemaType.builder(JsonSchemaPrimitiveUtil.JsonSchemaPrimitive.ARRAY)
|
||||
.withItems(item.asJsonSchemaType())
|
||||
.build()
|
||||
}
|
||||
|
||||
enum class LeafAirbyteType(
|
||||
enum class LeafAirbyteSchemaType(
|
||||
private val jsonSchemaType: JsonSchemaType,
|
||||
) : AirbyteType {
|
||||
) : AirbyteSchemaType {
|
||||
BOOLEAN(JsonSchemaType.BOOLEAN),
|
||||
STRING(JsonSchemaType.STRING),
|
||||
BINARY(JsonSchemaType.STRING_BASE_64),
|
||||
@@ -20,7 +20,7 @@ interface AirbyteStreamFactory {
|
||||
discoveredStream.id.name,
|
||||
discoveredStream.id.namespace,
|
||||
discoveredStream.columns.map {
|
||||
AirbyteField.of(it.id, it.type.airbyteType.asJsonSchemaType())
|
||||
AirbyteField.of(it.id, it.type.airbyteSchemaType.asJsonSchemaType())
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
|
||||
package io.airbyte.cdk.discover
|
||||
|
||||
import io.airbyte.cdk.data.AirbyteType
|
||||
import io.airbyte.cdk.data.AirbyteSchemaType
|
||||
import io.airbyte.cdk.data.IntCodec
|
||||
import io.airbyte.cdk.data.JsonDecoder
|
||||
import io.airbyte.cdk.data.JsonEncoder
|
||||
import io.airbyte.cdk.data.JsonStringCodec
|
||||
import io.airbyte.cdk.data.LeafAirbyteType
|
||||
import io.airbyte.cdk.data.LeafAirbyteSchemaType
|
||||
import io.airbyte.cdk.data.OffsetDateTimeCodec
|
||||
import java.time.OffsetDateTime
|
||||
|
||||
@@ -23,7 +23,7 @@ sealed interface FieldOrMetaField {
|
||||
*/
|
||||
interface FieldType {
|
||||
/** maps to [io.airbyte.protocol.models.Field.type] */
|
||||
val airbyteType: AirbyteType
|
||||
val airbyteSchemaType: AirbyteSchemaType
|
||||
val jsonEncoder: JsonEncoder<*>
|
||||
}
|
||||
|
||||
@@ -73,19 +73,20 @@ enum class CommonMetaField(
|
||||
}
|
||||
|
||||
data object CdcStringMetaFieldType : LosslessFieldType {
|
||||
override val airbyteType: AirbyteType = LeafAirbyteType.STRING
|
||||
override val airbyteSchemaType: AirbyteSchemaType = LeafAirbyteSchemaType.STRING
|
||||
override val jsonEncoder: JsonEncoder<String> = JsonStringCodec
|
||||
override val jsonDecoder: JsonDecoder<String> = JsonStringCodec
|
||||
}
|
||||
|
||||
data object CdcIntegerMetaFieldType : LosslessFieldType {
|
||||
override val airbyteType: AirbyteType = LeafAirbyteType.INTEGER
|
||||
override val airbyteSchemaType: AirbyteSchemaType = LeafAirbyteSchemaType.INTEGER
|
||||
override val jsonEncoder: JsonEncoder<Int> = IntCodec
|
||||
override val jsonDecoder: JsonDecoder<Int> = IntCodec
|
||||
}
|
||||
|
||||
data object CdcOffsetDateTimeMetaFieldType : LosslessFieldType {
|
||||
override val airbyteType: AirbyteType = LeafAirbyteType.TIMESTAMP_WITH_TIMEZONE
|
||||
override val airbyteSchemaType: AirbyteSchemaType =
|
||||
LeafAirbyteSchemaType.TIMESTAMP_WITH_TIMEZONE
|
||||
override val jsonEncoder: JsonEncoder<OffsetDateTime> = OffsetDateTimeCodec
|
||||
override val jsonDecoder: JsonDecoder<OffsetDateTime> = OffsetDateTimeCodec
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
package io.airbyte.cdk.output
|
||||
|
||||
import io.airbyte.cdk.StreamIdentifier
|
||||
import io.airbyte.cdk.data.AirbyteType
|
||||
import io.airbyte.cdk.data.AirbyteSchemaType
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.DefaultImplementation
|
||||
import jakarta.inject.Singleton
|
||||
@@ -43,8 +43,8 @@ data class FieldNotFound(
|
||||
data class FieldTypeMismatch(
|
||||
override val streamID: StreamIdentifier,
|
||||
val fieldName: String,
|
||||
val expected: AirbyteType,
|
||||
val actual: AirbyteType,
|
||||
val expected: AirbyteSchemaType,
|
||||
val actual: AirbyteSchemaType,
|
||||
) : CatalogValidationFailure
|
||||
|
||||
data class InvalidPrimaryKey(
|
||||
|
||||
@@ -10,9 +10,9 @@ import io.airbyte.cdk.command.GlobalInputState
|
||||
import io.airbyte.cdk.command.InputState
|
||||
import io.airbyte.cdk.command.SourceConfiguration
|
||||
import io.airbyte.cdk.command.StreamInputState
|
||||
import io.airbyte.cdk.data.AirbyteType
|
||||
import io.airbyte.cdk.data.ArrayAirbyteType
|
||||
import io.airbyte.cdk.data.LeafAirbyteType
|
||||
import io.airbyte.cdk.data.AirbyteSchemaType
|
||||
import io.airbyte.cdk.data.ArrayAirbyteSchemaType
|
||||
import io.airbyte.cdk.data.LeafAirbyteSchemaType
|
||||
import io.airbyte.cdk.discover.CommonMetaField
|
||||
import io.airbyte.cdk.discover.Field
|
||||
import io.airbyte.cdk.discover.FieldOrMetaField
|
||||
@@ -132,7 +132,7 @@ class StateManagerFactory(
|
||||
}
|
||||
}
|
||||
|
||||
val expectedSchema: Map<String, AirbyteType> =
|
||||
val expectedSchema: Map<String, AirbyteSchemaType> =
|
||||
jsonSchemaProperties.properties().associate { (id: String, schema: JsonNode) ->
|
||||
id to airbyteTypeFromJsonSchema(schema)
|
||||
}
|
||||
@@ -150,15 +150,15 @@ class StateManagerFactory(
|
||||
handler.accept(FieldNotFound(streamID, id))
|
||||
return null
|
||||
}
|
||||
val expectedAirbyteType: AirbyteType = expectedSchema[id] ?: return null
|
||||
val actualAirbyteType: AirbyteType = actualColumn.type.airbyteType
|
||||
if (expectedAirbyteType != actualAirbyteType) {
|
||||
val expectedAirbyteSchemaType: AirbyteSchemaType = expectedSchema[id] ?: return null
|
||||
val actualAirbyteSchemaType: AirbyteSchemaType = actualColumn.type.airbyteSchemaType
|
||||
if (expectedAirbyteSchemaType != actualAirbyteSchemaType) {
|
||||
handler.accept(
|
||||
FieldTypeMismatch(
|
||||
streamID,
|
||||
id,
|
||||
expectedAirbyteType,
|
||||
actualAirbyteType,
|
||||
expectedAirbyteSchemaType,
|
||||
actualAirbyteSchemaType,
|
||||
),
|
||||
)
|
||||
return null
|
||||
@@ -229,44 +229,44 @@ class StateManagerFactory(
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively re-generates the original [AirbyteType] from a catalog stream field's JSON
|
||||
* Recursively re-generates the original [AirbyteSchemaType] from a catalog stream field's JSON
|
||||
* schema.
|
||||
*/
|
||||
private fun airbyteTypeFromJsonSchema(jsonSchema: JsonNode): AirbyteType {
|
||||
private fun airbyteTypeFromJsonSchema(jsonSchema: JsonNode): AirbyteSchemaType {
|
||||
fun value(key: String): String = jsonSchema[key]?.asText() ?: ""
|
||||
return when (value("type")) {
|
||||
"array" -> ArrayAirbyteType(airbyteTypeFromJsonSchema(jsonSchema["items"]))
|
||||
"null" -> LeafAirbyteType.NULL
|
||||
"boolean" -> LeafAirbyteType.BOOLEAN
|
||||
"array" -> ArrayAirbyteSchemaType(airbyteTypeFromJsonSchema(jsonSchema["items"]))
|
||||
"null" -> LeafAirbyteSchemaType.NULL
|
||||
"boolean" -> LeafAirbyteSchemaType.BOOLEAN
|
||||
"number" ->
|
||||
when (value("airbyte_type")) {
|
||||
"integer",
|
||||
"big_integer", -> LeafAirbyteType.INTEGER
|
||||
else -> LeafAirbyteType.NUMBER
|
||||
"big_integer", -> LeafAirbyteSchemaType.INTEGER
|
||||
else -> LeafAirbyteSchemaType.NUMBER
|
||||
}
|
||||
"string" ->
|
||||
when (value("format")) {
|
||||
"date" -> LeafAirbyteType.DATE
|
||||
"date" -> LeafAirbyteSchemaType.DATE
|
||||
"date-time" ->
|
||||
if (value("airbyte_type") == "timestamp_with_timezone") {
|
||||
LeafAirbyteType.TIMESTAMP_WITH_TIMEZONE
|
||||
LeafAirbyteSchemaType.TIMESTAMP_WITH_TIMEZONE
|
||||
} else {
|
||||
LeafAirbyteType.TIMESTAMP_WITHOUT_TIMEZONE
|
||||
LeafAirbyteSchemaType.TIMESTAMP_WITHOUT_TIMEZONE
|
||||
}
|
||||
"time" ->
|
||||
if (value("airbyte_type") == "time_with_timezone") {
|
||||
LeafAirbyteType.TIME_WITH_TIMEZONE
|
||||
LeafAirbyteSchemaType.TIME_WITH_TIMEZONE
|
||||
} else {
|
||||
LeafAirbyteType.TIME_WITHOUT_TIMEZONE
|
||||
LeafAirbyteSchemaType.TIME_WITHOUT_TIMEZONE
|
||||
}
|
||||
else ->
|
||||
if (value("contentEncoding") == "base64") {
|
||||
LeafAirbyteType.BINARY
|
||||
LeafAirbyteSchemaType.BINARY
|
||||
} else {
|
||||
LeafAirbyteType.STRING
|
||||
LeafAirbyteSchemaType.STRING
|
||||
}
|
||||
}
|
||||
else -> LeafAirbyteType.JSONB
|
||||
else -> LeafAirbyteSchemaType.JSONB
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,12 +4,12 @@
|
||||
|
||||
package io.airbyte.cdk.discover
|
||||
|
||||
import io.airbyte.cdk.data.AirbyteType
|
||||
import io.airbyte.cdk.data.AirbyteSchemaType
|
||||
import io.airbyte.cdk.data.IntCodec
|
||||
import io.airbyte.cdk.data.JsonEncoder
|
||||
import io.airbyte.cdk.data.LeafAirbyteType
|
||||
import io.airbyte.cdk.data.LeafAirbyteSchemaType
|
||||
|
||||
data object IntFieldType : FieldType {
|
||||
override val airbyteType: AirbyteType = LeafAirbyteType.INTEGER
|
||||
override val airbyteSchemaType: AirbyteSchemaType = LeafAirbyteSchemaType.INTEGER
|
||||
override val jsonEncoder: JsonEncoder<*> = IntCodec
|
||||
}
|
||||
|
||||
@@ -4,12 +4,13 @@
|
||||
|
||||
package io.airbyte.cdk.discover
|
||||
|
||||
import io.airbyte.cdk.data.AirbyteType
|
||||
import io.airbyte.cdk.data.AirbyteSchemaType
|
||||
import io.airbyte.cdk.data.JsonEncoder
|
||||
import io.airbyte.cdk.data.LeafAirbyteType
|
||||
import io.airbyte.cdk.data.LeafAirbyteSchemaType
|
||||
import io.airbyte.cdk.data.OffsetDateTimeCodec
|
||||
|
||||
data object OffsetDateTimeFieldType : FieldType {
|
||||
override val airbyteType: AirbyteType = LeafAirbyteType.TIMESTAMP_WITH_TIMEZONE
|
||||
override val airbyteSchemaType: AirbyteSchemaType =
|
||||
LeafAirbyteSchemaType.TIMESTAMP_WITH_TIMEZONE
|
||||
override val jsonEncoder: JsonEncoder<*> = OffsetDateTimeCodec
|
||||
}
|
||||
|
||||
@@ -4,12 +4,12 @@
|
||||
|
||||
package io.airbyte.cdk.discover
|
||||
|
||||
import io.airbyte.cdk.data.AirbyteType
|
||||
import io.airbyte.cdk.data.AirbyteSchemaType
|
||||
import io.airbyte.cdk.data.JsonEncoder
|
||||
import io.airbyte.cdk.data.LeafAirbyteType
|
||||
import io.airbyte.cdk.data.LeafAirbyteSchemaType
|
||||
import io.airbyte.cdk.data.TextCodec
|
||||
|
||||
data object StringFieldType : FieldType {
|
||||
override val airbyteType: AirbyteType = LeafAirbyteType.STRING
|
||||
override val airbyteSchemaType: AirbyteSchemaType = LeafAirbyteSchemaType.STRING
|
||||
override val jsonEncoder: JsonEncoder<*> = TextCodec
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ class TestAirbyteStreamFactory : AirbyteStreamFactory {
|
||||
supportedSyncModes = listOf(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)
|
||||
(jsonSchema["properties"] as ObjectNode).apply {
|
||||
for (metaField in CommonMetaField.entries) {
|
||||
set<ObjectNode>(metaField.id, metaField.type.airbyteType.asJsonSchema())
|
||||
set<ObjectNode>(metaField.id, metaField.type.airbyteSchemaType.asJsonSchema())
|
||||
}
|
||||
}
|
||||
defaultCursorField = listOf(CommonMetaField.CDC_LSN.id)
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.mock_integration_test
|
||||
package io.airbyte.cdk.load.mock_integration_test
|
||||
|
||||
import io.airbyte.cdk.test.util.NoopDestinationCleaner
|
||||
import io.airbyte.cdk.test.util.NoopExpectedRecordMapper
|
||||
import io.airbyte.cdk.test.util.NoopNameMapper
|
||||
import io.airbyte.cdk.test.write.BasicFunctionalityIntegrationTest
|
||||
import io.airbyte.cdk.load.test.util.NoopDestinationCleaner
|
||||
import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper
|
||||
import io.airbyte.cdk.load.test.util.NoopNameMapper
|
||||
import io.airbyte.cdk.load.write.BasicFunctionalityIntegrationTest
|
||||
|
||||
class MockBasicFunctionalityIntegrationTest :
|
||||
BasicFunctionalityIntegrationTest(
|
||||
@@ -2,10 +2,10 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.mock_integration_test
|
||||
package io.airbyte.cdk.load.mock_integration_test
|
||||
|
||||
import io.airbyte.cdk.test.util.DestinationDataDumper
|
||||
import io.airbyte.cdk.test.util.OutputRecord
|
||||
import io.airbyte.cdk.load.test.util.DestinationDataDumper
|
||||
import io.airbyte.cdk.load.test.util.OutputRecord
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
object MockDestinationBackend {
|
||||
@@ -2,9 +2,9 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.mock_integration_test
|
||||
package io.airbyte.cdk.load.mock_integration_test
|
||||
|
||||
import io.airbyte.cdk.check.DestinationChecker
|
||||
import io.airbyte.cdk.load.check.DestinationChecker
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
@@ -2,11 +2,11 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.mock_integration_test
|
||||
package io.airbyte.cdk.load.mock_integration_test
|
||||
|
||||
import io.airbyte.cdk.command.ConfigurationSpecification
|
||||
import io.airbyte.cdk.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.command.DestinationConfigurationFactory
|
||||
import io.airbyte.cdk.load.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.load.command.DestinationConfigurationFactory
|
||||
import io.micronaut.context.annotation.Factory
|
||||
import jakarta.inject.Singleton
|
||||
|
||||
@@ -2,16 +2,16 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.mock_integration_test
|
||||
package io.airbyte.cdk.load.mock_integration_test
|
||||
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.data.ObjectValue
|
||||
import io.airbyte.cdk.message.Batch
|
||||
import io.airbyte.cdk.message.DestinationRecord
|
||||
import io.airbyte.cdk.message.SimpleBatch
|
||||
import io.airbyte.cdk.test.util.OutputRecord
|
||||
import io.airbyte.cdk.write.DestinationWriter
|
||||
import io.airbyte.cdk.write.StreamLoader
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.data.ObjectValue
|
||||
import io.airbyte.cdk.load.message.Batch
|
||||
import io.airbyte.cdk.load.message.DestinationRecord
|
||||
import io.airbyte.cdk.load.message.SimpleBatch
|
||||
import io.airbyte.cdk.load.test.util.OutputRecord
|
||||
import io.airbyte.cdk.load.write.DestinationWriter
|
||||
import io.airbyte.cdk.load.write.StreamLoader
|
||||
import java.time.Instant
|
||||
import java.util.UUID
|
||||
import javax.inject.Singleton
|
||||
@@ -2,13 +2,13 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.check
|
||||
package io.airbyte.cdk.load.check
|
||||
|
||||
import io.airbyte.cdk.Operation
|
||||
import io.airbyte.cdk.command.ConfigurationSpecification
|
||||
import io.airbyte.cdk.command.ConfigurationSpecificationSupplier
|
||||
import io.airbyte.cdk.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.command.DestinationConfigurationFactory
|
||||
import io.airbyte.cdk.load.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.load.command.DestinationConfigurationFactory
|
||||
import io.airbyte.cdk.output.ExceptionHandler
|
||||
import io.airbyte.cdk.output.OutputConsumer
|
||||
import io.airbyte.protocol.models.v0.AirbyteConnectionStatus
|
||||
@@ -2,9 +2,9 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.check
|
||||
package io.airbyte.cdk.load.check
|
||||
|
||||
import io.airbyte.cdk.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.load.command.DestinationConfiguration
|
||||
|
||||
/**
|
||||
* A check operation that is run before the destination is used.
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.command
|
||||
package io.airbyte.cdk.load.command
|
||||
|
||||
import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog
|
||||
import io.micronaut.context.annotation.Factory
|
||||
@@ -2,8 +2,11 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.command
|
||||
package io.airbyte.cdk.load.command
|
||||
|
||||
import io.airbyte.cdk.command.Configuration
|
||||
import io.airbyte.cdk.command.ConfigurationSpecification
|
||||
import io.airbyte.cdk.command.ConfigurationSpecificationSupplier
|
||||
import io.micronaut.context.annotation.Factory
|
||||
import jakarta.inject.Singleton
|
||||
import java.nio.file.Path
|
||||
@@ -2,9 +2,10 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.command
|
||||
package io.airbyte.cdk.load.command
|
||||
|
||||
import io.airbyte.cdk.ConfigErrorException
|
||||
import io.airbyte.cdk.command.ConfigurationSpecification
|
||||
|
||||
interface DestinationConfigurationFactory<
|
||||
I : ConfigurationSpecification, O : DestinationConfiguration> {
|
||||
@@ -2,11 +2,11 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.command
|
||||
package io.airbyte.cdk.load.command
|
||||
|
||||
import io.airbyte.cdk.data.AirbyteType
|
||||
import io.airbyte.cdk.data.AirbyteTypeToJsonSchema
|
||||
import io.airbyte.cdk.data.JsonSchemaToAirbyteType
|
||||
import io.airbyte.cdk.load.data.AirbyteType
|
||||
import io.airbyte.cdk.load.data.AirbyteTypeToJsonSchema
|
||||
import io.airbyte.cdk.load.data.JsonSchemaToAirbyteType
|
||||
import io.airbyte.protocol.models.v0.AirbyteStream
|
||||
import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream
|
||||
import io.airbyte.protocol.models.v0.DestinationSyncMode
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.data
|
||||
package io.airbyte.cdk.load.data
|
||||
|
||||
sealed interface AirbyteType
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.data
|
||||
package io.airbyte.cdk.load.data
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode
|
||||
import com.fasterxml.jackson.databind.node.JsonNodeFactory
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.data
|
||||
package io.airbyte.cdk.load.data
|
||||
|
||||
import java.math.BigDecimal
|
||||
import java.time.LocalDate
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.data
|
||||
package io.airbyte.cdk.load.data
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode
|
||||
import com.fasterxml.jackson.databind.node.JsonNodeFactory
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.data
|
||||
package io.airbyte.cdk.load.data
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode
|
||||
import com.fasterxml.jackson.databind.node.JsonNodeFactory
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.data
|
||||
package io.airbyte.cdk.load.data
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode
|
||||
import java.math.BigDecimal
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.file
|
||||
package io.airbyte.cdk.load.file
|
||||
|
||||
import io.micronaut.context.annotation.DefaultImplementation
|
||||
import java.io.Closeable
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.file
|
||||
package io.airbyte.cdk.load.file
|
||||
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.file
|
||||
package io.airbyte.cdk.load.file
|
||||
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,12 +2,12 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.message
|
||||
package io.airbyte.cdk.load.message
|
||||
|
||||
import com.google.common.collect.Range
|
||||
import com.google.common.collect.RangeSet
|
||||
import com.google.common.collect.TreeRangeSet
|
||||
import io.airbyte.cdk.file.LocalFile
|
||||
import io.airbyte.cdk.load.file.LocalFile
|
||||
|
||||
/**
|
||||
* Represents an accumulated batch of records in some stage of processing.
|
||||
@@ -2,17 +2,17 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.message
|
||||
package io.airbyte.cdk.load.message
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode
|
||||
import io.airbyte.cdk.command.DestinationCatalog
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.data.AirbyteValue
|
||||
import io.airbyte.cdk.data.AirbyteValueToJson
|
||||
import io.airbyte.cdk.data.JsonToAirbyteValue
|
||||
import io.airbyte.cdk.data.ObjectTypeWithoutSchema
|
||||
import io.airbyte.cdk.message.CheckpointMessage.Checkpoint
|
||||
import io.airbyte.cdk.message.CheckpointMessage.Stats
|
||||
import io.airbyte.cdk.load.command.DestinationCatalog
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.data.AirbyteValue
|
||||
import io.airbyte.cdk.load.data.AirbyteValueToJson
|
||||
import io.airbyte.cdk.load.data.JsonToAirbyteValue
|
||||
import io.airbyte.cdk.load.data.ObjectTypeWithoutSchema
|
||||
import io.airbyte.cdk.load.message.CheckpointMessage.Checkpoint
|
||||
import io.airbyte.cdk.load.message.CheckpointMessage.Stats
|
||||
import io.airbyte.protocol.models.Jsons
|
||||
import io.airbyte.protocol.models.v0.AirbyteGlobalState
|
||||
import io.airbyte.protocol.models.v0.AirbyteMessage
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.message
|
||||
package io.airbyte.cdk.load.message
|
||||
|
||||
import io.airbyte.cdk.util.Jsons
|
||||
import io.airbyte.protocol.models.v0.AirbyteMessage
|
||||
@@ -2,12 +2,12 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.message
|
||||
package io.airbyte.cdk.load.message
|
||||
|
||||
import io.airbyte.cdk.command.DestinationCatalog
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.state.MemoryManager
|
||||
import io.airbyte.cdk.state.Reserved
|
||||
import io.airbyte.cdk.load.command.DestinationCatalog
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.state.MemoryManager
|
||||
import io.airbyte.cdk.load.state.Reserved
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
@@ -2,9 +2,9 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.message
|
||||
package io.airbyte.cdk.load.message
|
||||
|
||||
import io.airbyte.cdk.util.CloseableCoroutine
|
||||
import io.airbyte.cdk.load.util.CloseableCoroutine
|
||||
import kotlinx.coroutines.channels.Channel
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.receiveAsFlow
|
||||
@@ -2,8 +2,10 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.spec
|
||||
package io.airbyte.cdk.load.spec
|
||||
|
||||
import io.airbyte.cdk.spec.IdentitySpecificationExtender
|
||||
import io.airbyte.cdk.spec.SpecificationExtender
|
||||
import io.airbyte.protocol.models.v0.ConnectorSpecification
|
||||
import io.airbyte.protocol.models.v0.DestinationSyncMode
|
||||
import io.micronaut.context.annotation.Replaces
|
||||
@@ -2,14 +2,14 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.state
|
||||
package io.airbyte.cdk.load.state
|
||||
|
||||
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
|
||||
import io.airbyte.cdk.command.DestinationCatalog
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.file.TimeProvider
|
||||
import io.airbyte.cdk.message.CheckpointMessage
|
||||
import io.airbyte.cdk.util.use
|
||||
import io.airbyte.cdk.load.command.DestinationCatalog
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.file.TimeProvider
|
||||
import io.airbyte.cdk.load.message.CheckpointMessage
|
||||
import io.airbyte.cdk.load.util.use
|
||||
import io.airbyte.protocol.models.v0.AirbyteMessage
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
@@ -2,14 +2,14 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.state
|
||||
package io.airbyte.cdk.load.state
|
||||
|
||||
import com.google.common.collect.Range
|
||||
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
|
||||
import io.airbyte.cdk.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.message.QueueReader
|
||||
import io.airbyte.cdk.task.internal.ForceFlushEvent
|
||||
import io.airbyte.cdk.load.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.message.QueueReader
|
||||
import io.airbyte.cdk.load.task.internal.ForceFlushEvent
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
@@ -2,9 +2,9 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.state
|
||||
package io.airbyte.cdk.load.state
|
||||
|
||||
import io.airbyte.cdk.util.CloseableCoroutine
|
||||
import io.airbyte.cdk.load.util.CloseableCoroutine
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
@@ -2,14 +2,14 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.state
|
||||
package io.airbyte.cdk.load.state
|
||||
|
||||
import com.google.common.collect.Range
|
||||
import com.google.common.collect.RangeSet
|
||||
import com.google.common.collect.TreeRangeSet
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.message.Batch
|
||||
import io.airbyte.cdk.message.BatchEnvelope
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.message.Batch
|
||||
import io.airbyte.cdk.load.message.BatchEnvelope
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,12 +2,12 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.state
|
||||
package io.airbyte.cdk.load.state
|
||||
|
||||
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
|
||||
import io.airbyte.cdk.command.DestinationCatalog
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.write.StreamLoader
|
||||
import io.airbyte.cdk.load.command.DestinationCatalog
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.write.StreamLoader
|
||||
import io.micronaut.context.annotation.Factory
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,16 +2,16 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task
|
||||
package io.airbyte.cdk.load.task
|
||||
|
||||
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
|
||||
import io.airbyte.cdk.command.DestinationCatalog
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.state.StreamSucceeded
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.state.SyncSuccess
|
||||
import io.airbyte.cdk.task.implementor.FailStreamTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.FailSyncTaskFactory
|
||||
import io.airbyte.cdk.load.command.DestinationCatalog
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.state.StreamSucceeded
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.state.SyncSuccess
|
||||
import io.airbyte.cdk.load.task.implementor.FailStreamTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.FailSyncTaskFactory
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,26 +2,26 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task
|
||||
package io.airbyte.cdk.load.task
|
||||
|
||||
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
|
||||
import io.airbyte.cdk.command.DestinationCatalog
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.message.Batch
|
||||
import io.airbyte.cdk.message.BatchEnvelope
|
||||
import io.airbyte.cdk.message.SpilledRawMessagesLocalFile
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.task.implementor.CloseStreamTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.OpenStreamTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.ProcessBatchTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.ProcessRecordsTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.SetupTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.TeardownTaskFactory
|
||||
import io.airbyte.cdk.task.internal.FlushCheckpointsTaskFactory
|
||||
import io.airbyte.cdk.task.internal.InputConsumerTask
|
||||
import io.airbyte.cdk.task.internal.SpillToDiskTaskFactory
|
||||
import io.airbyte.cdk.task.internal.TimedForcedCheckpointFlushTask
|
||||
import io.airbyte.cdk.task.internal.UpdateCheckpointsTask
|
||||
import io.airbyte.cdk.load.command.DestinationCatalog
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.message.Batch
|
||||
import io.airbyte.cdk.load.message.BatchEnvelope
|
||||
import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.task.implementor.CloseStreamTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.OpenStreamTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.ProcessBatchTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.ProcessRecordsTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.SetupTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.TeardownTaskFactory
|
||||
import io.airbyte.cdk.load.task.internal.FlushCheckpointsTaskFactory
|
||||
import io.airbyte.cdk.load.task.internal.InputConsumerTask
|
||||
import io.airbyte.cdk.load.task.internal.SpillToDiskTaskFactory
|
||||
import io.airbyte.cdk.load.task.internal.TimedForcedCheckpointFlushTask
|
||||
import io.airbyte.cdk.load.task.internal.UpdateCheckpointsTask
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,9 +2,9 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task
|
||||
package io.airbyte.cdk.load.task
|
||||
|
||||
import io.airbyte.cdk.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.load.command.DestinationConfiguration
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,9 +2,9 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task
|
||||
package io.airbyte.cdk.load.task
|
||||
|
||||
import io.airbyte.cdk.util.CloseableCoroutine
|
||||
import io.airbyte.cdk.load.util.CloseableCoroutine
|
||||
|
||||
interface Task {
|
||||
suspend fun execute()
|
||||
@@ -2,14 +2,14 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.implementor
|
||||
package io.airbyte.cdk.load.task.implementor
|
||||
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.task.ImplementorTask
|
||||
import io.airbyte.cdk.task.StreamTask
|
||||
import io.airbyte.cdk.write.StreamLoader
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.load.task.ImplementorTask
|
||||
import io.airbyte.cdk.load.task.StreamTask
|
||||
import io.airbyte.cdk.load.write.StreamLoader
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.implementor
|
||||
package io.airbyte.cdk.load.task.implementor
|
||||
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.state.StreamIncompleteResult
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.task.DestinationTaskExceptionHandler
|
||||
import io.airbyte.cdk.task.ImplementorTask
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.state.StreamIncompleteResult
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.task.DestinationTaskExceptionHandler
|
||||
import io.airbyte.cdk.load.task.ImplementorTask
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,13 +2,13 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.implementor
|
||||
package io.airbyte.cdk.load.task.implementor
|
||||
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.task.DestinationTaskExceptionHandler
|
||||
import io.airbyte.cdk.task.ImplementorTask
|
||||
import io.airbyte.cdk.util.setOnce
|
||||
import io.airbyte.cdk.write.DestinationWriter
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.task.DestinationTaskExceptionHandler
|
||||
import io.airbyte.cdk.load.task.ImplementorTask
|
||||
import io.airbyte.cdk.load.util.setOnce
|
||||
import io.airbyte.cdk.load.write.DestinationWriter
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,15 +2,15 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.implementor
|
||||
package io.airbyte.cdk.load.task.implementor
|
||||
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.task.ImplementorTask
|
||||
import io.airbyte.cdk.task.StreamTask
|
||||
import io.airbyte.cdk.write.DestinationWriter
|
||||
import io.airbyte.cdk.write.StreamLoader
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.load.task.ImplementorTask
|
||||
import io.airbyte.cdk.load.task.StreamTask
|
||||
import io.airbyte.cdk.load.write.DestinationWriter
|
||||
import io.airbyte.cdk.load.write.StreamLoader
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
|
||||
@@ -2,15 +2,15 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.implementor
|
||||
package io.airbyte.cdk.load.task.implementor
|
||||
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.message.BatchEnvelope
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.task.ImplementorTask
|
||||
import io.airbyte.cdk.task.StreamTask
|
||||
import io.airbyte.cdk.write.StreamLoader
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.message.BatchEnvelope
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.load.task.ImplementorTask
|
||||
import io.airbyte.cdk.load.task.StreamTask
|
||||
import io.airbyte.cdk.load.write.StreamLoader
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
|
||||
@@ -2,22 +2,22 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.implementor
|
||||
package io.airbyte.cdk.load.task.implementor
|
||||
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.message.BatchEnvelope
|
||||
import io.airbyte.cdk.message.Deserializer
|
||||
import io.airbyte.cdk.message.DestinationMessage
|
||||
import io.airbyte.cdk.message.DestinationRecord
|
||||
import io.airbyte.cdk.message.DestinationStreamAffinedMessage
|
||||
import io.airbyte.cdk.message.DestinationStreamComplete
|
||||
import io.airbyte.cdk.message.DestinationStreamIncomplete
|
||||
import io.airbyte.cdk.message.SpilledRawMessagesLocalFile
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.task.ImplementorTask
|
||||
import io.airbyte.cdk.task.StreamTask
|
||||
import io.airbyte.cdk.write.StreamLoader
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.message.BatchEnvelope
|
||||
import io.airbyte.cdk.load.message.Deserializer
|
||||
import io.airbyte.cdk.load.message.DestinationMessage
|
||||
import io.airbyte.cdk.load.message.DestinationRecord
|
||||
import io.airbyte.cdk.load.message.DestinationStreamAffinedMessage
|
||||
import io.airbyte.cdk.load.message.DestinationStreamComplete
|
||||
import io.airbyte.cdk.load.message.DestinationStreamIncomplete
|
||||
import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.load.task.ImplementorTask
|
||||
import io.airbyte.cdk.load.task.StreamTask
|
||||
import io.airbyte.cdk.load.write.StreamLoader
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,12 +2,12 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.implementor
|
||||
package io.airbyte.cdk.load.task.implementor
|
||||
|
||||
import io.airbyte.cdk.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.task.ImplementorTask
|
||||
import io.airbyte.cdk.task.SyncTask
|
||||
import io.airbyte.cdk.write.DestinationWriter
|
||||
import io.airbyte.cdk.load.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.load.task.ImplementorTask
|
||||
import io.airbyte.cdk.load.task.SyncTask
|
||||
import io.airbyte.cdk.load.write.DestinationWriter
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.implementor
|
||||
package io.airbyte.cdk.load.task.implementor
|
||||
|
||||
import io.airbyte.cdk.state.CheckpointManager
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.task.SyncTask
|
||||
import io.airbyte.cdk.write.DestinationWriter
|
||||
import io.airbyte.cdk.load.state.CheckpointManager
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.load.task.SyncTask
|
||||
import io.airbyte.cdk.load.write.DestinationWriter
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,11 +2,11 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.internal
|
||||
package io.airbyte.cdk.load.task.internal
|
||||
|
||||
import io.airbyte.cdk.state.CheckpointManager
|
||||
import io.airbyte.cdk.task.InternalTask
|
||||
import io.airbyte.cdk.task.SyncTask
|
||||
import io.airbyte.cdk.load.state.CheckpointManager
|
||||
import io.airbyte.cdk.load.task.InternalTask
|
||||
import io.airbyte.cdk.load.task.SyncTask
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
|
||||
@@ -2,36 +2,36 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.internal
|
||||
package io.airbyte.cdk.load.task.internal
|
||||
|
||||
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
|
||||
import io.airbyte.cdk.command.DestinationCatalog
|
||||
import io.airbyte.cdk.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.message.CheckpointMessage
|
||||
import io.airbyte.cdk.message.CheckpointMessageWrapped
|
||||
import io.airbyte.cdk.message.Deserializer
|
||||
import io.airbyte.cdk.message.DestinationMessage
|
||||
import io.airbyte.cdk.message.DestinationRecord
|
||||
import io.airbyte.cdk.message.DestinationRecordWrapped
|
||||
import io.airbyte.cdk.message.DestinationStreamAffinedMessage
|
||||
import io.airbyte.cdk.message.DestinationStreamComplete
|
||||
import io.airbyte.cdk.message.DestinationStreamIncomplete
|
||||
import io.airbyte.cdk.message.GlobalCheckpoint
|
||||
import io.airbyte.cdk.message.GlobalCheckpointWrapped
|
||||
import io.airbyte.cdk.message.MessageQueueSupplier
|
||||
import io.airbyte.cdk.message.QueueWriter
|
||||
import io.airbyte.cdk.message.StreamCheckpoint
|
||||
import io.airbyte.cdk.message.StreamCheckpointWrapped
|
||||
import io.airbyte.cdk.message.StreamCompleteWrapped
|
||||
import io.airbyte.cdk.message.StreamRecordWrapped
|
||||
import io.airbyte.cdk.message.Undefined
|
||||
import io.airbyte.cdk.state.MemoryManager
|
||||
import io.airbyte.cdk.state.Reserved
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.task.InternalTask
|
||||
import io.airbyte.cdk.task.SyncTask
|
||||
import io.airbyte.cdk.util.use
|
||||
import io.airbyte.cdk.load.command.DestinationCatalog
|
||||
import io.airbyte.cdk.load.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.message.CheckpointMessage
|
||||
import io.airbyte.cdk.load.message.CheckpointMessageWrapped
|
||||
import io.airbyte.cdk.load.message.Deserializer
|
||||
import io.airbyte.cdk.load.message.DestinationMessage
|
||||
import io.airbyte.cdk.load.message.DestinationRecord
|
||||
import io.airbyte.cdk.load.message.DestinationRecordWrapped
|
||||
import io.airbyte.cdk.load.message.DestinationStreamAffinedMessage
|
||||
import io.airbyte.cdk.load.message.DestinationStreamComplete
|
||||
import io.airbyte.cdk.load.message.DestinationStreamIncomplete
|
||||
import io.airbyte.cdk.load.message.GlobalCheckpoint
|
||||
import io.airbyte.cdk.load.message.GlobalCheckpointWrapped
|
||||
import io.airbyte.cdk.load.message.MessageQueueSupplier
|
||||
import io.airbyte.cdk.load.message.QueueWriter
|
||||
import io.airbyte.cdk.load.message.StreamCheckpoint
|
||||
import io.airbyte.cdk.load.message.StreamCheckpointWrapped
|
||||
import io.airbyte.cdk.load.message.StreamCompleteWrapped
|
||||
import io.airbyte.cdk.load.message.StreamRecordWrapped
|
||||
import io.airbyte.cdk.load.message.Undefined
|
||||
import io.airbyte.cdk.load.state.MemoryManager
|
||||
import io.airbyte.cdk.load.state.Reserved
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.task.InternalTask
|
||||
import io.airbyte.cdk.load.task.SyncTask
|
||||
import io.airbyte.cdk.load.util.use
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,27 +2,27 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.internal
|
||||
package io.airbyte.cdk.load.task.internal
|
||||
|
||||
import com.google.common.collect.Range
|
||||
import io.airbyte.cdk.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.file.TempFileProvider
|
||||
import io.airbyte.cdk.message.BatchEnvelope
|
||||
import io.airbyte.cdk.message.DestinationRecordWrapped
|
||||
import io.airbyte.cdk.message.MessageQueueSupplier
|
||||
import io.airbyte.cdk.message.QueueReader
|
||||
import io.airbyte.cdk.message.SpilledRawMessagesLocalFile
|
||||
import io.airbyte.cdk.message.StreamCompleteWrapped
|
||||
import io.airbyte.cdk.message.StreamRecordWrapped
|
||||
import io.airbyte.cdk.state.FlushStrategy
|
||||
import io.airbyte.cdk.state.Reserved
|
||||
import io.airbyte.cdk.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.task.InternalTask
|
||||
import io.airbyte.cdk.task.StreamTask
|
||||
import io.airbyte.cdk.util.takeUntilInclusive
|
||||
import io.airbyte.cdk.util.use
|
||||
import io.airbyte.cdk.util.withNextAdjacentValue
|
||||
import io.airbyte.cdk.load.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.file.TempFileProvider
|
||||
import io.airbyte.cdk.load.message.BatchEnvelope
|
||||
import io.airbyte.cdk.load.message.DestinationRecordWrapped
|
||||
import io.airbyte.cdk.load.message.MessageQueueSupplier
|
||||
import io.airbyte.cdk.load.message.QueueReader
|
||||
import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile
|
||||
import io.airbyte.cdk.load.message.StreamCompleteWrapped
|
||||
import io.airbyte.cdk.load.message.StreamRecordWrapped
|
||||
import io.airbyte.cdk.load.state.FlushStrategy
|
||||
import io.airbyte.cdk.load.state.Reserved
|
||||
import io.airbyte.cdk.load.task.DestinationTaskLauncher
|
||||
import io.airbyte.cdk.load.task.InternalTask
|
||||
import io.airbyte.cdk.load.task.StreamTask
|
||||
import io.airbyte.cdk.load.util.takeUntilInclusive
|
||||
import io.airbyte.cdk.load.util.use
|
||||
import io.airbyte.cdk.load.util.withNextAdjacentValue
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import jakarta.inject.Singleton
|
||||
import kotlinx.coroutines.flow.last
|
||||
@@ -2,16 +2,16 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.internal
|
||||
package io.airbyte.cdk.load.task.internal
|
||||
|
||||
import io.airbyte.cdk.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.file.TimeProvider
|
||||
import io.airbyte.cdk.message.ChannelMessageQueue
|
||||
import io.airbyte.cdk.message.QueueWriter
|
||||
import io.airbyte.cdk.state.CheckpointManager
|
||||
import io.airbyte.cdk.task.SyncTask
|
||||
import io.airbyte.cdk.util.use
|
||||
import io.airbyte.cdk.load.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.file.TimeProvider
|
||||
import io.airbyte.cdk.load.message.ChannelMessageQueue
|
||||
import io.airbyte.cdk.load.message.QueueWriter
|
||||
import io.airbyte.cdk.load.state.CheckpointManager
|
||||
import io.airbyte.cdk.load.task.SyncTask
|
||||
import io.airbyte.cdk.load.util.use
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,17 +2,17 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.internal
|
||||
package io.airbyte.cdk.load.task.internal
|
||||
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.message.CheckpointMessage
|
||||
import io.airbyte.cdk.message.CheckpointMessageWrapped
|
||||
import io.airbyte.cdk.message.GlobalCheckpointWrapped
|
||||
import io.airbyte.cdk.message.MessageQueue
|
||||
import io.airbyte.cdk.message.StreamCheckpointWrapped
|
||||
import io.airbyte.cdk.state.CheckpointManager
|
||||
import io.airbyte.cdk.state.Reserved
|
||||
import io.airbyte.cdk.task.SyncTask
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.message.CheckpointMessage
|
||||
import io.airbyte.cdk.load.message.CheckpointMessageWrapped
|
||||
import io.airbyte.cdk.load.message.GlobalCheckpointWrapped
|
||||
import io.airbyte.cdk.load.message.MessageQueue
|
||||
import io.airbyte.cdk.load.message.StreamCheckpointWrapped
|
||||
import io.airbyte.cdk.load.state.CheckpointManager
|
||||
import io.airbyte.cdk.load.state.Reserved
|
||||
import io.airbyte.cdk.load.task.SyncTask
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.util
|
||||
package io.airbyte.cdk.load.util
|
||||
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.util
|
||||
package io.airbyte.cdk.load.util
|
||||
|
||||
import com.google.common.collect.Range
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.write
|
||||
package io.airbyte.cdk.load.write
|
||||
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.state.SyncFailure
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.state.SyncFailure
|
||||
import io.micronaut.context.annotation.Secondary
|
||||
import jakarta.inject.Singleton
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.write
|
||||
package io.airbyte.cdk.load.write
|
||||
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.message.Batch
|
||||
import io.airbyte.cdk.message.DestinationRecord
|
||||
import io.airbyte.cdk.message.SimpleBatch
|
||||
import io.airbyte.cdk.state.StreamIncompleteResult
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.message.Batch
|
||||
import io.airbyte.cdk.load.message.DestinationRecord
|
||||
import io.airbyte.cdk.load.message.SimpleBatch
|
||||
import io.airbyte.cdk.load.state.StreamIncompleteResult
|
||||
|
||||
/**
|
||||
* Implementor interface. The framework calls open and close once per stream at the beginning and
|
||||
@@ -2,13 +2,13 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.write
|
||||
package io.airbyte.cdk.load.write
|
||||
|
||||
import io.airbyte.cdk.Operation
|
||||
import io.airbyte.cdk.state.SyncFailure
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.state.SyncSuccess
|
||||
import io.airbyte.cdk.task.TaskLauncher
|
||||
import io.airbyte.cdk.load.state.SyncFailure
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.state.SyncSuccess
|
||||
import io.airbyte.cdk.load.task.TaskLauncher
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import io.micronaut.context.annotation.Factory
|
||||
import io.micronaut.context.annotation.Requires
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.command
|
||||
package io.airbyte.cdk.load.command
|
||||
|
||||
import io.airbyte.protocol.models.Jsons
|
||||
import io.airbyte.protocol.models.v0.AirbyteStream
|
||||
@@ -2,12 +2,12 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.command
|
||||
package io.airbyte.cdk.load.command
|
||||
|
||||
import io.airbyte.cdk.data.FieldType
|
||||
import io.airbyte.cdk.data.IntegerType
|
||||
import io.airbyte.cdk.data.ObjectType
|
||||
import io.airbyte.cdk.data.StringType
|
||||
import io.airbyte.cdk.load.data.FieldType
|
||||
import io.airbyte.cdk.load.data.IntegerType
|
||||
import io.airbyte.cdk.load.data.ObjectType
|
||||
import io.airbyte.cdk.load.data.StringType
|
||||
import io.micronaut.context.annotation.Factory
|
||||
import io.micronaut.context.annotation.Primary
|
||||
import io.micronaut.context.annotation.Requires
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.command
|
||||
package io.airbyte.cdk.load.command
|
||||
|
||||
import io.micronaut.context.annotation.Primary
|
||||
import io.micronaut.context.annotation.Requires
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.data
|
||||
package io.airbyte.cdk.load.data
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode
|
||||
import com.fasterxml.jackson.databind.node.JsonNodeFactory
|
||||
@@ -10,7 +10,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode
|
||||
import org.junit.jupiter.api.Assertions
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class AirbyteTypeToJsonSchemaTest {
|
||||
class AirbyteSchemaTypeToJsonSchemaTest {
|
||||
@Test
|
||||
fun testRoundTrip() {
|
||||
val schema = JsonNodeFactory.instance.objectNode()
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.data
|
||||
package io.airbyte.cdk.load.data
|
||||
|
||||
import org.junit.jupiter.api.Assertions
|
||||
import org.junit.jupiter.api.Test
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.data
|
||||
package io.airbyte.cdk.load.data
|
||||
|
||||
import com.fasterxml.jackson.databind.node.JsonNodeFactory
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode
|
||||
@@ -10,7 +10,7 @@ import io.airbyte.cdk.util.Jsons
|
||||
import org.junit.jupiter.api.Assertions
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class JsonSchemaToAirbyteTypeTest {
|
||||
class JsonSchemaToAirbyteSchemaTypeTest {
|
||||
private fun ofType(type: String): ObjectNode {
|
||||
return JsonNodeFactory.instance.objectNode().put("type", type)
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.data
|
||||
package io.airbyte.cdk.load.data
|
||||
|
||||
import com.fasterxml.jackson.databind.node.JsonNodeFactory
|
||||
import java.math.BigDecimal
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.file
|
||||
package io.airbyte.cdk.load.file
|
||||
|
||||
import io.micronaut.context.annotation.Requires
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.file
|
||||
package io.airbyte.cdk.load.file
|
||||
|
||||
import io.micronaut.context.annotation.Primary
|
||||
import io.micronaut.context.annotation.Requires
|
||||
@@ -2,12 +2,12 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.message
|
||||
package io.airbyte.cdk.load.message
|
||||
|
||||
import io.airbyte.cdk.command.Append
|
||||
import io.airbyte.cdk.command.DestinationCatalog
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.data.ObjectTypeWithEmptySchema
|
||||
import io.airbyte.cdk.load.command.Append
|
||||
import io.airbyte.cdk.load.command.DestinationCatalog
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.data.ObjectTypeWithEmptySchema
|
||||
import io.airbyte.protocol.models.Jsons
|
||||
import io.airbyte.protocol.models.v0.AirbyteGlobalState
|
||||
import io.airbyte.protocol.models.v0.AirbyteMessage
|
||||
@@ -19,7 +19,7 @@ import io.airbyte.protocol.models.v0.AirbyteStateStats
|
||||
import io.airbyte.protocol.models.v0.AirbyteStreamState
|
||||
import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage
|
||||
import io.airbyte.protocol.models.v0.AirbyteTraceMessage
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.params.ParameterizedTest
|
||||
import org.junit.jupiter.params.provider.Arguments
|
||||
@@ -47,7 +47,7 @@ class DestinationMessageTest {
|
||||
fun testRoundTrip(message: AirbyteMessage) {
|
||||
val roundTripped =
|
||||
factory.fromAirbyteMessage(message, Jsons.serialize(message)).asProtocolMessage()
|
||||
assertEquals(message, roundTripped)
|
||||
Assertions.assertEquals(message, roundTripped)
|
||||
}
|
||||
|
||||
// Checkpoint messages aren't round-trippable.
|
||||
@@ -73,7 +73,7 @@ class DestinationMessageTest {
|
||||
factory.fromAirbyteMessage(inputMessage, Jsons.serialize(inputMessage))
|
||||
as StreamCheckpoint
|
||||
|
||||
assertEquals(
|
||||
Assertions.assertEquals(
|
||||
inputMessage.also {
|
||||
it.state.destinationStats = AirbyteStateStats().withRecordCount(3.0)
|
||||
},
|
||||
@@ -110,7 +110,7 @@ class DestinationMessageTest {
|
||||
Jsons.serialize(inputMessage),
|
||||
) as GlobalCheckpoint
|
||||
|
||||
assertEquals(
|
||||
Assertions.assertEquals(
|
||||
inputMessage.also {
|
||||
it.state.destinationStats = AirbyteStateStats().withRecordCount(3.0)
|
||||
},
|
||||
@@ -2,18 +2,18 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.state
|
||||
package io.airbyte.cdk.load.state
|
||||
|
||||
import com.google.common.collect.Range
|
||||
import com.google.common.collect.TreeRangeSet
|
||||
import io.airbyte.cdk.command.DestinationCatalog
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream2
|
||||
import io.airbyte.cdk.file.TimeProvider
|
||||
import io.airbyte.cdk.message.Batch
|
||||
import io.airbyte.cdk.message.BatchEnvelope
|
||||
import io.airbyte.cdk.message.SimpleBatch
|
||||
import io.airbyte.cdk.load.command.DestinationCatalog
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream1
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream2
|
||||
import io.airbyte.cdk.load.file.TimeProvider
|
||||
import io.airbyte.cdk.load.message.Batch
|
||||
import io.airbyte.cdk.load.message.BatchEnvelope
|
||||
import io.airbyte.cdk.load.message.SimpleBatch
|
||||
import io.micronaut.context.annotation.Requires
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest
|
||||
import jakarta.inject.Inject
|
||||
@@ -2,14 +2,14 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.state
|
||||
package io.airbyte.cdk.load.state
|
||||
|
||||
import com.google.common.collect.Range
|
||||
import io.airbyte.cdk.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream2
|
||||
import io.airbyte.cdk.message.ChannelMessageQueue
|
||||
import io.airbyte.cdk.task.internal.ForceFlushEvent
|
||||
import io.airbyte.cdk.load.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream2
|
||||
import io.airbyte.cdk.load.message.ChannelMessageQueue
|
||||
import io.airbyte.cdk.load.task.internal.ForceFlushEvent
|
||||
import io.micronaut.context.annotation.Primary
|
||||
import io.micronaut.context.annotation.Requires
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.state
|
||||
package io.airbyte.cdk.load.state
|
||||
|
||||
import io.micronaut.context.annotation.Replaces
|
||||
import io.micronaut.context.annotation.Requires
|
||||
@@ -2,11 +2,11 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.state
|
||||
package io.airbyte.cdk.load.state
|
||||
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.file.TimeProvider
|
||||
import io.airbyte.cdk.message.CheckpointMessage
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.file.TimeProvider
|
||||
import io.airbyte.cdk.load.message.CheckpointMessage
|
||||
import io.micronaut.context.annotation.Requires
|
||||
import jakarta.inject.Inject
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,15 +2,15 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.state
|
||||
package io.airbyte.cdk.load.state
|
||||
|
||||
import com.google.common.collect.Range
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream2
|
||||
import io.airbyte.cdk.message.Batch
|
||||
import io.airbyte.cdk.message.BatchEnvelope
|
||||
import io.airbyte.cdk.message.SimpleBatch
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream1
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream2
|
||||
import io.airbyte.cdk.load.message.Batch
|
||||
import io.airbyte.cdk.load.message.BatchEnvelope
|
||||
import io.airbyte.cdk.load.message.SimpleBatch
|
||||
import java.util.stream.Stream
|
||||
import kotlinx.coroutines.channels.Channel
|
||||
import kotlinx.coroutines.delay
|
||||
@@ -2,12 +2,12 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.state
|
||||
package io.airbyte.cdk.load.state
|
||||
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream2
|
||||
import io.airbyte.cdk.test.util.CoroutineTestUtils
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream1
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory.Companion.stream2
|
||||
import io.airbyte.cdk.load.test.util.CoroutineTestUtils
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest
|
||||
import jakarta.inject.Inject
|
||||
import kotlinx.coroutines.channels.Channel
|
||||
@@ -2,13 +2,13 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.state
|
||||
package io.airbyte.cdk.load.state
|
||||
|
||||
import com.google.common.collect.Range
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.message.Batch
|
||||
import io.airbyte.cdk.message.BatchEnvelope
|
||||
import io.airbyte.cdk.message.SimpleBatch
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.message.Batch
|
||||
import io.airbyte.cdk.load.message.BatchEnvelope
|
||||
import io.airbyte.cdk.load.message.SimpleBatch
|
||||
|
||||
/**
|
||||
* Because [SyncManager] and [StreamManager] have thin interfaces with no side effects, mocking them
|
||||
@@ -2,17 +2,17 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task
|
||||
package io.airbyte.cdk.load.task
|
||||
|
||||
import io.airbyte.cdk.command.DestinationCatalog
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.task.implementor.FailStreamTask
|
||||
import io.airbyte.cdk.task.implementor.FailStreamTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.FailSyncTask
|
||||
import io.airbyte.cdk.task.implementor.FailSyncTaskFactory
|
||||
import io.airbyte.cdk.test.util.CoroutineTestUtils
|
||||
import io.airbyte.cdk.load.command.DestinationCatalog
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.task.implementor.FailStreamTask
|
||||
import io.airbyte.cdk.load.task.implementor.FailStreamTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.FailSyncTask
|
||||
import io.airbyte.cdk.load.task.implementor.FailSyncTaskFactory
|
||||
import io.airbyte.cdk.load.test.util.CoroutineTestUtils
|
||||
import io.micronaut.context.annotation.Primary
|
||||
import io.micronaut.context.annotation.Requires
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest
|
||||
@@ -87,7 +87,7 @@ class DestinationTaskExceptionHandlerTest {
|
||||
runTest {
|
||||
val mockTask =
|
||||
object : StreamTask {
|
||||
override val stream: DestinationStream = stream1
|
||||
override val stream: DestinationStream = MockDestinationCatalogFactory.stream1
|
||||
|
||||
override suspend fun execute() {
|
||||
throw RuntimeException("StreamTask failure")
|
||||
@@ -97,7 +97,7 @@ class DestinationTaskExceptionHandlerTest {
|
||||
val wrappedTask = exceptionHandler.withExceptionHandling(mockTask)
|
||||
wrappedTask.execute()
|
||||
val (stream, exception) = mockFailStreamTaskFactory.didRunFor.receive()
|
||||
Assertions.assertEquals(stream1, stream)
|
||||
Assertions.assertEquals(MockDestinationCatalogFactory.stream1, stream)
|
||||
Assertions.assertTrue(exception is RuntimeException)
|
||||
}
|
||||
|
||||
@@ -182,7 +182,7 @@ class DestinationTaskExceptionHandlerTest {
|
||||
val innerTaskRan = Channel<Boolean>(Channel.UNLIMITED)
|
||||
val mockTask =
|
||||
object : StreamTask {
|
||||
override val stream: DestinationStream = stream1
|
||||
override val stream: DestinationStream = MockDestinationCatalogFactory.stream1
|
||||
|
||||
override suspend fun execute() {
|
||||
innerTaskRan.send(true)
|
||||
@@ -190,7 +190,7 @@ class DestinationTaskExceptionHandlerTest {
|
||||
}
|
||||
|
||||
val wrappedTask = exceptionHandler.withExceptionHandling(mockTask)
|
||||
val manager = syncManager.getStreamManager(stream1.descriptor)
|
||||
val manager = syncManager.getStreamManager(MockDestinationCatalogFactory.stream1.descriptor)
|
||||
manager.markEndOfStream()
|
||||
manager.markFailed(RuntimeException("dummy failure"))
|
||||
launch { wrappedTask.execute() }
|
||||
@@ -207,7 +207,7 @@ class DestinationTaskExceptionHandlerTest {
|
||||
) = runTest {
|
||||
val mockTask =
|
||||
object : StreamTask {
|
||||
override val stream: DestinationStream = stream1
|
||||
override val stream: DestinationStream = MockDestinationCatalogFactory.stream1
|
||||
|
||||
override suspend fun execute() {
|
||||
// do nothing
|
||||
@@ -216,7 +216,7 @@ class DestinationTaskExceptionHandlerTest {
|
||||
|
||||
val wrappedTask = exceptionHandler.withExceptionHandling(mockTask)
|
||||
|
||||
val manager = syncManager.getStreamManager(stream1.descriptor)
|
||||
val manager = syncManager.getStreamManager(MockDestinationCatalogFactory.stream1.descriptor)
|
||||
manager.markEndOfStream()
|
||||
manager.markSucceeded()
|
||||
|
||||
@@ -2,44 +2,44 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task
|
||||
package io.airbyte.cdk.load.task
|
||||
|
||||
import com.google.common.collect.Range
|
||||
import com.google.common.collect.TreeRangeSet
|
||||
import io.airbyte.cdk.command.DestinationCatalog
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1
|
||||
import io.airbyte.cdk.file.DefaultLocalFile
|
||||
import io.airbyte.cdk.message.Batch
|
||||
import io.airbyte.cdk.message.BatchEnvelope
|
||||
import io.airbyte.cdk.message.SpilledRawMessagesLocalFile
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.task.implementor.CloseStreamTask
|
||||
import io.airbyte.cdk.task.implementor.CloseStreamTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.DefaultCloseStreamTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.DefaultOpenStreamTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.DefaultProcessBatchTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.DefaultProcessRecordsTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.DefaultSetupTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.DefaultTeardownTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.OpenStreamTask
|
||||
import io.airbyte.cdk.task.implementor.OpenStreamTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.ProcessBatchTask
|
||||
import io.airbyte.cdk.task.implementor.ProcessBatchTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.ProcessRecordsTask
|
||||
import io.airbyte.cdk.task.implementor.ProcessRecordsTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.SetupTask
|
||||
import io.airbyte.cdk.task.implementor.SetupTaskFactory
|
||||
import io.airbyte.cdk.task.implementor.TeardownTask
|
||||
import io.airbyte.cdk.task.implementor.TeardownTaskFactory
|
||||
import io.airbyte.cdk.task.internal.DefaultSpillToDiskTaskFactory
|
||||
import io.airbyte.cdk.task.internal.FlushCheckpointsTask
|
||||
import io.airbyte.cdk.task.internal.FlushCheckpointsTaskFactory
|
||||
import io.airbyte.cdk.task.internal.InputConsumerTask
|
||||
import io.airbyte.cdk.task.internal.SpillToDiskTask
|
||||
import io.airbyte.cdk.task.internal.SpillToDiskTaskFactory
|
||||
import io.airbyte.cdk.task.internal.TimedForcedCheckpointFlushTask
|
||||
import io.airbyte.cdk.task.internal.UpdateCheckpointsTask
|
||||
import io.airbyte.cdk.load.command.DestinationCatalog
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory
|
||||
import io.airbyte.cdk.load.file.DefaultLocalFile
|
||||
import io.airbyte.cdk.load.message.Batch
|
||||
import io.airbyte.cdk.load.message.BatchEnvelope
|
||||
import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.task.implementor.CloseStreamTask
|
||||
import io.airbyte.cdk.load.task.implementor.CloseStreamTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.DefaultCloseStreamTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.DefaultOpenStreamTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.DefaultProcessBatchTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.DefaultProcessRecordsTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.DefaultSetupTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.DefaultTeardownTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.OpenStreamTask
|
||||
import io.airbyte.cdk.load.task.implementor.OpenStreamTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.ProcessBatchTask
|
||||
import io.airbyte.cdk.load.task.implementor.ProcessBatchTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.ProcessRecordsTask
|
||||
import io.airbyte.cdk.load.task.implementor.ProcessRecordsTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.SetupTask
|
||||
import io.airbyte.cdk.load.task.implementor.SetupTaskFactory
|
||||
import io.airbyte.cdk.load.task.implementor.TeardownTask
|
||||
import io.airbyte.cdk.load.task.implementor.TeardownTaskFactory
|
||||
import io.airbyte.cdk.load.task.internal.DefaultSpillToDiskTaskFactory
|
||||
import io.airbyte.cdk.load.task.internal.FlushCheckpointsTask
|
||||
import io.airbyte.cdk.load.task.internal.FlushCheckpointsTaskFactory
|
||||
import io.airbyte.cdk.load.task.internal.InputConsumerTask
|
||||
import io.airbyte.cdk.load.task.internal.SpillToDiskTask
|
||||
import io.airbyte.cdk.load.task.internal.SpillToDiskTaskFactory
|
||||
import io.airbyte.cdk.load.task.internal.TimedForcedCheckpointFlushTask
|
||||
import io.airbyte.cdk.load.task.internal.UpdateCheckpointsTask
|
||||
import io.micronaut.context.annotation.Primary
|
||||
import io.micronaut.context.annotation.Replaces
|
||||
import io.micronaut.context.annotation.Requires
|
||||
@@ -337,7 +337,7 @@ class DestinationTaskLauncherTest {
|
||||
@Test
|
||||
fun testHandleSpilledFileCompleteNotEndOfStream() = runTest {
|
||||
taskLauncher.handleNewSpilledFile(
|
||||
stream1,
|
||||
MockDestinationCatalogFactory.stream1,
|
||||
BatchEnvelope(
|
||||
SpilledRawMessagesLocalFile(DefaultLocalFile(Path("not/a/real/file")), 100L)
|
||||
),
|
||||
@@ -345,7 +345,8 @@ class DestinationTaskLauncherTest {
|
||||
)
|
||||
|
||||
processRecordsTaskFactory.hasRun.receive()
|
||||
mockSpillToDiskTaskFactory.streamHasRun[stream1.descriptor]?.receive()
|
||||
mockSpillToDiskTaskFactory.streamHasRun[MockDestinationCatalogFactory.stream1.descriptor]
|
||||
?.receive()
|
||||
?: Assertions.fail("SpillToDiskTask not run")
|
||||
}
|
||||
|
||||
@@ -353,7 +354,7 @@ class DestinationTaskLauncherTest {
|
||||
fun testHandleSpilledFileCompleteEndOfStream() = runTest {
|
||||
launch {
|
||||
taskLauncher.handleNewSpilledFile(
|
||||
stream1,
|
||||
MockDestinationCatalogFactory.stream1,
|
||||
BatchEnvelope(
|
||||
SpilledRawMessagesLocalFile(DefaultLocalFile(Path("not/a/real/file")), 100L)
|
||||
),
|
||||
@@ -364,22 +365,25 @@ class DestinationTaskLauncherTest {
|
||||
processRecordsTaskFactory.hasRun.receive()
|
||||
delay(500)
|
||||
Assertions.assertTrue(
|
||||
mockSpillToDiskTaskFactory.streamHasRun[stream1.descriptor]?.tryReceive()?.isFailure !=
|
||||
false
|
||||
mockSpillToDiskTaskFactory.streamHasRun[
|
||||
MockDestinationCatalogFactory.stream1.descriptor]
|
||||
?.tryReceive()
|
||||
?.isFailure != false
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testHandleNewBatch() = runTest {
|
||||
val range = TreeRangeSet.create(listOf(Range.closed(0L, 100L)))
|
||||
val streamManager = syncManager.getStreamManager(stream1.descriptor)
|
||||
val streamManager =
|
||||
syncManager.getStreamManager(MockDestinationCatalogFactory.stream1.descriptor)
|
||||
repeat(100) { streamManager.countRecordIn() }
|
||||
|
||||
streamManager.markEndOfStream()
|
||||
|
||||
// Verify incomplete batch triggers process batch
|
||||
val incompleteBatch = BatchEnvelope(MockBatch(Batch.State.LOCAL), range)
|
||||
taskLauncher.handleNewBatch(stream1, incompleteBatch)
|
||||
taskLauncher.handleNewBatch(MockDestinationCatalogFactory.stream1, incompleteBatch)
|
||||
Assertions.assertFalse(streamManager.areRecordsPersistedUntil(100L))
|
||||
|
||||
val batchReceived = processBatchTaskFactory.hasRun.receive()
|
||||
@@ -388,21 +392,21 @@ class DestinationTaskLauncherTest {
|
||||
Assertions.assertTrue(flushCheckpointsTaskFactory.hasRun.tryReceive().isFailure)
|
||||
|
||||
val persistedBatch = BatchEnvelope(MockBatch(Batch.State.PERSISTED), range)
|
||||
taskLauncher.handleNewBatch(stream1, persistedBatch)
|
||||
taskLauncher.handleNewBatch(MockDestinationCatalogFactory.stream1, persistedBatch)
|
||||
Assertions.assertTrue(streamManager.areRecordsPersistedUntil(100L))
|
||||
Assertions.assertTrue(flushCheckpointsTaskFactory.hasRun.receive())
|
||||
|
||||
// Verify complete batch w/o batch processing complete does nothing
|
||||
val halfRange = TreeRangeSet.create(listOf(Range.closed(0L, 50L)))
|
||||
val completeBatchHalf = BatchEnvelope(MockBatch(Batch.State.COMPLETE), halfRange)
|
||||
taskLauncher.handleNewBatch(stream1, completeBatchHalf)
|
||||
taskLauncher.handleNewBatch(MockDestinationCatalogFactory.stream1, completeBatchHalf)
|
||||
delay(1000)
|
||||
Assertions.assertTrue(closeStreamTaskFactory.hasRun.tryReceive().isFailure)
|
||||
|
||||
// Verify complete batch w/ batch processing complete triggers close stream
|
||||
val secondHalf = TreeRangeSet.create(listOf(Range.closed(51L, 100L)))
|
||||
val completingBatch = BatchEnvelope(MockBatch(Batch.State.COMPLETE), secondHalf)
|
||||
taskLauncher.handleNewBatch(stream1, completingBatch)
|
||||
taskLauncher.handleNewBatch(MockDestinationCatalogFactory.stream1, completingBatch)
|
||||
closeStreamTaskFactory.hasRun.receive()
|
||||
Assertions.assertTrue(true)
|
||||
}
|
||||
@@ -410,7 +414,7 @@ class DestinationTaskLauncherTest {
|
||||
@Test
|
||||
fun testHandleStreamClosed() = runTest {
|
||||
// This should run teardown unconditionally.
|
||||
launch { taskLauncher.handleStreamClosed(stream1) }
|
||||
launch { taskLauncher.handleStreamClosed(MockDestinationCatalogFactory.stream1) }
|
||||
teardownTaskFactory.hasRun.receive()
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task
|
||||
package io.airbyte.cdk.load.task
|
||||
|
||||
import io.micronaut.context.annotation.Primary
|
||||
import io.micronaut.context.annotation.Requires
|
||||
@@ -2,11 +2,11 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task
|
||||
package io.airbyte.cdk.load.task
|
||||
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.message.BatchEnvelope
|
||||
import io.airbyte.cdk.message.SpilledRawMessagesLocalFile
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.message.BatchEnvelope
|
||||
import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile
|
||||
import io.micronaut.context.annotation.Primary
|
||||
import io.micronaut.context.annotation.Requires
|
||||
import jakarta.inject.Singleton
|
||||
@@ -2,22 +2,22 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.implementor
|
||||
package io.airbyte.cdk.load.task.implementor
|
||||
|
||||
import com.google.common.collect.Range
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1
|
||||
import io.airbyte.cdk.data.IntegerValue
|
||||
import io.airbyte.cdk.file.MockTempFileProvider
|
||||
import io.airbyte.cdk.message.Batch
|
||||
import io.airbyte.cdk.message.BatchEnvelope
|
||||
import io.airbyte.cdk.message.Deserializer
|
||||
import io.airbyte.cdk.message.DestinationMessage
|
||||
import io.airbyte.cdk.message.DestinationRecord
|
||||
import io.airbyte.cdk.message.SpilledRawMessagesLocalFile
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.task.MockTaskLauncher
|
||||
import io.airbyte.cdk.write.StreamLoader
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory
|
||||
import io.airbyte.cdk.load.data.IntegerValue
|
||||
import io.airbyte.cdk.load.file.MockTempFileProvider
|
||||
import io.airbyte.cdk.load.message.Batch
|
||||
import io.airbyte.cdk.load.message.BatchEnvelope
|
||||
import io.airbyte.cdk.load.message.Deserializer
|
||||
import io.airbyte.cdk.load.message.DestinationMessage
|
||||
import io.airbyte.cdk.load.message.DestinationRecord
|
||||
import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.task.MockTaskLauncher
|
||||
import io.airbyte.cdk.load.write.StreamLoader
|
||||
import io.micronaut.context.annotation.Primary
|
||||
import io.micronaut.context.annotation.Requires
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest
|
||||
@@ -49,7 +49,7 @@ class ProcessRecordsTaskTest {
|
||||
) : Batch
|
||||
|
||||
class MockStreamLoader : StreamLoader {
|
||||
override val stream: DestinationStream = stream1
|
||||
override val stream: DestinationStream = MockDestinationCatalogFactory.stream1
|
||||
|
||||
data class SumAndCount(val sum: Long = 0, val count: Long = 0)
|
||||
|
||||
@@ -78,7 +78,7 @@ class ProcessRecordsTaskTest {
|
||||
class MockDeserializer : Deserializer<DestinationMessage> {
|
||||
override fun deserialize(serialized: String): DestinationMessage {
|
||||
return DestinationRecord(
|
||||
stream = stream1.descriptor,
|
||||
stream = MockDestinationCatalogFactory.stream1.descriptor,
|
||||
data = IntegerValue(serialized.toLong()),
|
||||
emittedAtMs = 0L,
|
||||
meta = null,
|
||||
@@ -104,7 +104,7 @@ class ProcessRecordsTaskTest {
|
||||
val task =
|
||||
processRecordsTaskFactory.make(
|
||||
taskLauncher = launcher,
|
||||
stream = stream1,
|
||||
stream = MockDestinationCatalogFactory.stream1,
|
||||
fileEnvelope = BatchEnvelope(file, Range.closed(0, 1024))
|
||||
)
|
||||
mockFile.linesToRead = (0 until recordCount).map { "$it" }.toMutableList()
|
||||
@@ -2,34 +2,33 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.internal
|
||||
package io.airbyte.cdk.load.task.internal
|
||||
|
||||
import com.fasterxml.jackson.databind.node.JsonNodeFactory
|
||||
import io.airbyte.cdk.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream2
|
||||
import io.airbyte.cdk.data.NullValue
|
||||
import io.airbyte.cdk.message.CheckpointMessage
|
||||
import io.airbyte.cdk.message.CheckpointMessageWrapped
|
||||
import io.airbyte.cdk.message.DestinationMessage
|
||||
import io.airbyte.cdk.message.DestinationRecord
|
||||
import io.airbyte.cdk.message.DestinationRecordWrapped
|
||||
import io.airbyte.cdk.message.DestinationStreamComplete
|
||||
import io.airbyte.cdk.message.DestinationStreamIncomplete
|
||||
import io.airbyte.cdk.message.GlobalCheckpoint
|
||||
import io.airbyte.cdk.message.GlobalCheckpointWrapped
|
||||
import io.airbyte.cdk.message.MessageQueue
|
||||
import io.airbyte.cdk.message.MessageQueueSupplier
|
||||
import io.airbyte.cdk.message.StreamCheckpoint
|
||||
import io.airbyte.cdk.message.StreamCheckpointWrapped
|
||||
import io.airbyte.cdk.message.StreamCompleteWrapped
|
||||
import io.airbyte.cdk.message.StreamRecordWrapped
|
||||
import io.airbyte.cdk.state.MemoryManager
|
||||
import io.airbyte.cdk.state.Reserved
|
||||
import io.airbyte.cdk.state.SyncManager
|
||||
import io.airbyte.cdk.test.util.CoroutineTestUtils
|
||||
import io.airbyte.cdk.util.takeUntilInclusive
|
||||
import io.airbyte.cdk.load.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory
|
||||
import io.airbyte.cdk.load.data.NullValue
|
||||
import io.airbyte.cdk.load.message.CheckpointMessage
|
||||
import io.airbyte.cdk.load.message.CheckpointMessageWrapped
|
||||
import io.airbyte.cdk.load.message.DestinationMessage
|
||||
import io.airbyte.cdk.load.message.DestinationRecord
|
||||
import io.airbyte.cdk.load.message.DestinationRecordWrapped
|
||||
import io.airbyte.cdk.load.message.DestinationStreamComplete
|
||||
import io.airbyte.cdk.load.message.DestinationStreamIncomplete
|
||||
import io.airbyte.cdk.load.message.GlobalCheckpoint
|
||||
import io.airbyte.cdk.load.message.GlobalCheckpointWrapped
|
||||
import io.airbyte.cdk.load.message.MessageQueue
|
||||
import io.airbyte.cdk.load.message.MessageQueueSupplier
|
||||
import io.airbyte.cdk.load.message.StreamCheckpoint
|
||||
import io.airbyte.cdk.load.message.StreamCheckpointWrapped
|
||||
import io.airbyte.cdk.load.message.StreamCompleteWrapped
|
||||
import io.airbyte.cdk.load.message.StreamRecordWrapped
|
||||
import io.airbyte.cdk.load.state.MemoryManager
|
||||
import io.airbyte.cdk.load.state.Reserved
|
||||
import io.airbyte.cdk.load.state.SyncManager
|
||||
import io.airbyte.cdk.load.test.util.CoroutineTestUtils
|
||||
import io.airbyte.cdk.load.util.takeUntilInclusive
|
||||
import io.micronaut.context.annotation.Primary
|
||||
import io.micronaut.context.annotation.Requires
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest
|
||||
@@ -128,13 +127,20 @@ class InputConsumerTaskTest {
|
||||
|
||||
@Test
|
||||
fun testSendRecords() = runTest {
|
||||
val queue1 = recordQueueSupplier.get(stream1.descriptor)
|
||||
val queue2 = recordQueueSupplier.get(stream2.descriptor)
|
||||
val queue1 = recordQueueSupplier.get(MockDestinationCatalogFactory.stream1.descriptor)
|
||||
val queue2 = recordQueueSupplier.get(MockDestinationCatalogFactory.stream2.descriptor)
|
||||
|
||||
val manager1 = syncManager.getStreamManager(stream1.descriptor)
|
||||
val manager2 = syncManager.getStreamManager(stream2.descriptor)
|
||||
val manager1 =
|
||||
syncManager.getStreamManager(MockDestinationCatalogFactory.stream1.descriptor)
|
||||
val manager2 =
|
||||
syncManager.getStreamManager(MockDestinationCatalogFactory.stream2.descriptor)
|
||||
|
||||
(0 until 10).forEach { mockInputFlow.addMessage(makeRecord(stream1, "test${it}"), it * 2L) }
|
||||
(0 until 10).forEach {
|
||||
mockInputFlow.addMessage(
|
||||
makeRecord(MockDestinationCatalogFactory.stream1, "test${it}"),
|
||||
it * 2L
|
||||
)
|
||||
}
|
||||
launch { task.execute() }
|
||||
|
||||
val messages1 =
|
||||
@@ -148,7 +154,11 @@ class InputConsumerTaskTest {
|
||||
Assertions.assertEquals(10, messages1.size)
|
||||
val expectedRecords =
|
||||
(0 until 10).map {
|
||||
StreamRecordWrapped(it.toLong(), it * 2L, makeRecord(stream1, "test${it}"))
|
||||
StreamRecordWrapped(
|
||||
it.toLong(),
|
||||
it * 2L,
|
||||
makeRecord(MockDestinationCatalogFactory.stream1, "test${it}")
|
||||
)
|
||||
}
|
||||
|
||||
Assertions.assertEquals(expectedRecords, messages1.map { it.value })
|
||||
@@ -165,22 +175,35 @@ class InputConsumerTaskTest {
|
||||
|
||||
@Test
|
||||
fun testSendEndOfStream() = runTest {
|
||||
val queue1 = recordQueueSupplier.get(stream1.descriptor)
|
||||
val queue2 = recordQueueSupplier.get(stream2.descriptor)
|
||||
val queue1 = recordQueueSupplier.get(MockDestinationCatalogFactory.stream1.descriptor)
|
||||
val queue2 = recordQueueSupplier.get(MockDestinationCatalogFactory.stream2.descriptor)
|
||||
|
||||
val manager1 = syncManager.getStreamManager(stream1.descriptor)
|
||||
val manager2 = syncManager.getStreamManager(stream2.descriptor)
|
||||
val manager1 =
|
||||
syncManager.getStreamManager(MockDestinationCatalogFactory.stream1.descriptor)
|
||||
val manager2 =
|
||||
syncManager.getStreamManager(MockDestinationCatalogFactory.stream2.descriptor)
|
||||
|
||||
(0 until 10).forEach { _ -> mockInputFlow.addMessage(makeRecord(stream1, "whatever"), 0L) }
|
||||
(0 until 10).forEach { _ ->
|
||||
mockInputFlow.addMessage(
|
||||
makeRecord(MockDestinationCatalogFactory.stream1, "whatever"),
|
||||
0L
|
||||
)
|
||||
}
|
||||
|
||||
mockInputFlow.addMessage(makeRecord(stream2, "test"), 1L)
|
||||
mockInputFlow.addMessage(makeStreamComplete(stream1), 0L)
|
||||
mockInputFlow.addMessage(makeRecord(MockDestinationCatalogFactory.stream2, "test"), 1L)
|
||||
mockInputFlow.addMessage(makeStreamComplete(MockDestinationCatalogFactory.stream1), 0L)
|
||||
val job = launch { task.execute() }
|
||||
mockInputFlow.stop()
|
||||
job.join()
|
||||
queue2.close()
|
||||
Assertions.assertEquals(
|
||||
listOf(StreamRecordWrapped(0, 1L, makeRecord(stream2, "test"))),
|
||||
listOf(
|
||||
StreamRecordWrapped(
|
||||
0,
|
||||
1L,
|
||||
makeRecord(MockDestinationCatalogFactory.stream2, "test")
|
||||
)
|
||||
),
|
||||
queue2.consume().toList().map { it.value }
|
||||
)
|
||||
Assertions.assertEquals(1L, manager2.recordCount())
|
||||
@@ -209,10 +232,10 @@ class InputConsumerTaskTest {
|
||||
|
||||
val batches =
|
||||
listOf(
|
||||
TestEvent(stream1, 10, 10),
|
||||
TestEvent(stream1, 5, 15),
|
||||
TestEvent(stream2, 4, 4),
|
||||
TestEvent(stream1, 3, 18),
|
||||
TestEvent(MockDestinationCatalogFactory.stream1, 10, 10),
|
||||
TestEvent(MockDestinationCatalogFactory.stream1, 5, 15),
|
||||
TestEvent(MockDestinationCatalogFactory.stream2, 4, 4),
|
||||
TestEvent(MockDestinationCatalogFactory.stream1, 3, 18),
|
||||
)
|
||||
|
||||
launch { task.execute() }
|
||||
@@ -239,12 +262,12 @@ class InputConsumerTaskTest {
|
||||
|
||||
val batches =
|
||||
listOf(
|
||||
AddRecords(stream1, 10),
|
||||
AddRecords(MockDestinationCatalogFactory.stream1, 10),
|
||||
SendState(10, 0, 10),
|
||||
AddRecords(stream2, 5),
|
||||
AddRecords(stream1, 4),
|
||||
AddRecords(MockDestinationCatalogFactory.stream2, 5),
|
||||
AddRecords(MockDestinationCatalogFactory.stream1, 4),
|
||||
SendState(14, 5, 9),
|
||||
AddRecords(stream2, 3),
|
||||
AddRecords(MockDestinationCatalogFactory.stream2, 3),
|
||||
SendState(14, 8, 3),
|
||||
SendState(14, 8, 0),
|
||||
)
|
||||
@@ -262,8 +285,14 @@ class InputConsumerTaskTest {
|
||||
val state =
|
||||
checkpointQueue.consume().take(1).toList().first().value
|
||||
as GlobalCheckpointWrapped
|
||||
val stream1State = state.streamIndexes.find { it.first == stream1.descriptor }!!
|
||||
val stream2State = state.streamIndexes.find { it.first == stream2.descriptor }!!
|
||||
val stream1State =
|
||||
state.streamIndexes.find {
|
||||
it.first == MockDestinationCatalogFactory.stream1.descriptor
|
||||
}!!
|
||||
val stream2State =
|
||||
state.streamIndexes.find {
|
||||
it.first == MockDestinationCatalogFactory.stream2.descriptor
|
||||
}!!
|
||||
Assertions.assertEquals(event.expectedStream1Count, stream1State.second)
|
||||
Assertions.assertEquals(event.expectedStream2Count, stream2State.second)
|
||||
Assertions.assertEquals(
|
||||
@@ -278,8 +307,8 @@ class InputConsumerTaskTest {
|
||||
|
||||
@Test
|
||||
fun testStreamIncompleteThrows() = runTest {
|
||||
mockInputFlow.addMessage(makeRecord(stream1, "test"), 1L)
|
||||
mockInputFlow.addMessage(makeStreamIncomplete(stream1), 0L)
|
||||
mockInputFlow.addMessage(makeRecord(MockDestinationCatalogFactory.stream1, "test"), 1L)
|
||||
mockInputFlow.addMessage(makeStreamIncomplete(MockDestinationCatalogFactory.stream1), 0L)
|
||||
CoroutineTestUtils.assertThrows(IllegalStateException::class) { task.execute() }
|
||||
mockInputFlow.stop()
|
||||
}
|
||||
@@ -2,11 +2,11 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.internal
|
||||
package io.airbyte.cdk.load.task.internal
|
||||
|
||||
import io.airbyte.cdk.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.message.Deserializer
|
||||
import io.airbyte.cdk.state.MemoryManager
|
||||
import io.airbyte.cdk.load.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.load.message.Deserializer
|
||||
import io.airbyte.cdk.load.state.MemoryManager
|
||||
import io.micronaut.context.annotation.Primary
|
||||
import io.micronaut.context.annotation.Requires
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest
|
||||
@@ -2,22 +2,22 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.internal
|
||||
package io.airbyte.cdk.load.task.internal
|
||||
|
||||
import com.google.common.collect.Range
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.command.MockDestinationCatalogFactory.Companion.stream1
|
||||
import io.airbyte.cdk.data.NullValue
|
||||
import io.airbyte.cdk.file.MockTempFileProvider
|
||||
import io.airbyte.cdk.message.DestinationRecord
|
||||
import io.airbyte.cdk.message.DestinationRecordWrapped
|
||||
import io.airbyte.cdk.message.MessageQueueSupplier
|
||||
import io.airbyte.cdk.message.StreamCompleteWrapped
|
||||
import io.airbyte.cdk.message.StreamRecordWrapped
|
||||
import io.airbyte.cdk.state.FlushStrategy
|
||||
import io.airbyte.cdk.state.MemoryManager
|
||||
import io.airbyte.cdk.state.Reserved
|
||||
import io.airbyte.cdk.task.MockTaskLauncher
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.command.MockDestinationCatalogFactory
|
||||
import io.airbyte.cdk.load.data.NullValue
|
||||
import io.airbyte.cdk.load.file.MockTempFileProvider
|
||||
import io.airbyte.cdk.load.message.DestinationRecord
|
||||
import io.airbyte.cdk.load.message.DestinationRecordWrapped
|
||||
import io.airbyte.cdk.load.message.MessageQueueSupplier
|
||||
import io.airbyte.cdk.load.message.StreamCompleteWrapped
|
||||
import io.airbyte.cdk.load.message.StreamRecordWrapped
|
||||
import io.airbyte.cdk.load.state.FlushStrategy
|
||||
import io.airbyte.cdk.load.state.MemoryManager
|
||||
import io.airbyte.cdk.load.state.Reserved
|
||||
import io.airbyte.cdk.load.task.MockTaskLauncher
|
||||
import io.micronaut.context.annotation.Primary
|
||||
import io.micronaut.context.annotation.Requires
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest
|
||||
@@ -59,7 +59,7 @@ class SpillToDiskTaskTest {
|
||||
}
|
||||
|
||||
private suspend fun primeMessageQueue(): Long {
|
||||
val queue = queueSupplier.get(stream1.descriptor)
|
||||
val queue = queueSupplier.get(MockDestinationCatalogFactory.stream1.descriptor)
|
||||
val maxRecords = ((1024 * 1.5) / 8).toLong()
|
||||
var recordsWritten = 0L
|
||||
var bytesReserved = 0L
|
||||
@@ -74,7 +74,7 @@ class SpillToDiskTaskTest {
|
||||
sizeBytes = 8,
|
||||
record =
|
||||
DestinationRecord(
|
||||
stream = stream1.descriptor,
|
||||
stream = MockDestinationCatalogFactory.stream1.descriptor,
|
||||
data = NullValue,
|
||||
emittedAtMs = 0,
|
||||
meta = null,
|
||||
@@ -95,9 +95,13 @@ class SpillToDiskTaskTest {
|
||||
Assertions.assertEquals(availableMemory - bytesReserved, memoryManager.remainingMemoryBytes)
|
||||
|
||||
val mockTaskLauncher = MockTaskLauncher()
|
||||
spillToDiskTaskFactory.make(mockTaskLauncher, stream1).execute()
|
||||
spillToDiskTaskFactory
|
||||
.make(mockTaskLauncher, MockDestinationCatalogFactory.stream1)
|
||||
.execute()
|
||||
Assertions.assertEquals(1, mockTaskLauncher.spilledFiles.size)
|
||||
spillToDiskTaskFactory.make(mockTaskLauncher, stream1).execute()
|
||||
spillToDiskTaskFactory
|
||||
.make(mockTaskLauncher, MockDestinationCatalogFactory.stream1)
|
||||
.execute()
|
||||
Assertions.assertEquals(2, mockTaskLauncher.spilledFiles.size)
|
||||
|
||||
Assertions.assertEquals(1024, mockTaskLauncher.spilledFiles[0].batch.totalSizeBytes)
|
||||
@@ -2,14 +2,14 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.task.internal
|
||||
package io.airbyte.cdk.load.task.internal
|
||||
|
||||
import io.airbyte.cdk.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.file.MockTimeProvider
|
||||
import io.airbyte.cdk.message.QueueReader
|
||||
import io.airbyte.cdk.state.MockCheckpointManager
|
||||
import io.airbyte.cdk.task.MockTaskLauncher
|
||||
import io.airbyte.cdk.load.command.DestinationConfiguration
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.file.MockTimeProvider
|
||||
import io.airbyte.cdk.load.message.QueueReader
|
||||
import io.airbyte.cdk.load.state.MockCheckpointManager
|
||||
import io.airbyte.cdk.load.task.MockTaskLauncher
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest
|
||||
import jakarta.inject.Inject
|
||||
import kotlinx.coroutines.test.runTest
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.test.util
|
||||
package io.airbyte.cdk.load.test.util
|
||||
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.test.util
|
||||
package io.airbyte.cdk.load.test.util
|
||||
|
||||
import java.time.OffsetDateTime
|
||||
import org.junit.jupiter.api.Assertions
|
||||
@@ -2,15 +2,15 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.test.check
|
||||
package io.airbyte.cdk.load.check
|
||||
|
||||
import io.airbyte.cdk.command.ConfigurationSpecification
|
||||
import io.airbyte.cdk.command.ValidatedJsonUtils
|
||||
import io.airbyte.cdk.test.util.FakeDataDumper
|
||||
import io.airbyte.cdk.test.util.IntegrationTest
|
||||
import io.airbyte.cdk.test.util.NoopDestinationCleaner
|
||||
import io.airbyte.cdk.test.util.NoopExpectedRecordMapper
|
||||
import io.airbyte.cdk.test.util.destination_process.TestDeploymentMode
|
||||
import io.airbyte.cdk.load.test.util.FakeDataDumper
|
||||
import io.airbyte.cdk.load.test.util.IntegrationTest
|
||||
import io.airbyte.cdk.load.test.util.NoopDestinationCleaner
|
||||
import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper
|
||||
import io.airbyte.cdk.load.test.util.destination_process.TestDeploymentMode
|
||||
import io.airbyte.protocol.models.v0.AirbyteConnectionStatus
|
||||
import io.airbyte.protocol.models.v0.AirbyteMessage
|
||||
import java.nio.charset.StandardCharsets
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.test.spec
|
||||
package io.airbyte.cdk.load.spec
|
||||
|
||||
import com.deblock.jsondiff.DiffGenerator
|
||||
import com.deblock.jsondiff.diff.JsonDiff
|
||||
@@ -12,17 +12,16 @@ import com.deblock.jsondiff.matcher.LenientJsonObjectPartialMatcher
|
||||
import com.deblock.jsondiff.matcher.StrictJsonArrayPartialMatcher
|
||||
import com.deblock.jsondiff.matcher.StrictPrimitivePartialMatcher
|
||||
import com.deblock.jsondiff.viewer.OnlyErrorDiffViewer
|
||||
import io.airbyte.cdk.test.util.FakeDataDumper
|
||||
import io.airbyte.cdk.test.util.IntegrationTest
|
||||
import io.airbyte.cdk.test.util.NoopDestinationCleaner
|
||||
import io.airbyte.cdk.test.util.NoopExpectedRecordMapper
|
||||
import io.airbyte.cdk.test.util.destination_process.DestinationProcessFactory
|
||||
import io.airbyte.cdk.test.util.destination_process.TestDeploymentMode
|
||||
import io.airbyte.cdk.load.test.util.FakeDataDumper
|
||||
import io.airbyte.cdk.load.test.util.IntegrationTest
|
||||
import io.airbyte.cdk.load.test.util.NoopDestinationCleaner
|
||||
import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper
|
||||
import io.airbyte.cdk.load.test.util.destination_process.TestDeploymentMode
|
||||
import io.airbyte.cdk.util.Jsons
|
||||
import io.airbyte.protocol.models.v0.AirbyteMessage
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.api.assertAll
|
||||
|
||||
@@ -67,7 +66,7 @@ abstract class SpecTest :
|
||||
val messages = process.readMessages()
|
||||
val specMessages = messages.filter { it.type == AirbyteMessage.Type.SPEC }
|
||||
|
||||
assertEquals(
|
||||
Assertions.assertEquals(
|
||||
specMessages.size,
|
||||
1,
|
||||
"Expected to receive exactly one connection status message, but got ${specMessages.size}: $specMessages"
|
||||
@@ -88,8 +87,8 @@ abstract class SpecTest :
|
||||
DiffGenerator.diff(expectedSpec, Jsons.writeValueAsString(spec), jsonMatcher)
|
||||
assertAll(
|
||||
"Spec snapshot test failed. Run this test locally and then `git diff <...>/$expectedSpecFilename` to see what changed, and commit the diff if that change was intentional.",
|
||||
{ assertEquals("", OnlyErrorDiffViewer.from(diff).toString()) },
|
||||
{ assertEquals(expectedSpec, actualSpecPrettyPrint) }
|
||||
{ Assertions.assertEquals("", OnlyErrorDiffViewer.from(diff).toString()) },
|
||||
{ Assertions.assertEquals(expectedSpec, actualSpecPrettyPrint) }
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.test.util
|
||||
package io.airbyte.cdk.load.test.util
|
||||
|
||||
fun interface DestinationCleaner {
|
||||
/**
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.test.util
|
||||
package io.airbyte.cdk.load.test.util
|
||||
|
||||
fun interface DestinationDataDumper {
|
||||
fun dumpRecords(
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.test.util
|
||||
package io.airbyte.cdk.load.test.util
|
||||
|
||||
fun interface ExpectedRecordMapper {
|
||||
fun mapRecord(expectedRecord: OutputRecord): OutputRecord
|
||||
@@ -2,14 +2,14 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.test.util
|
||||
package io.airbyte.cdk.load.test.util
|
||||
|
||||
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
|
||||
import io.airbyte.cdk.command.ConfigurationSpecification
|
||||
import io.airbyte.cdk.command.DestinationCatalog
|
||||
import io.airbyte.cdk.command.DestinationStream
|
||||
import io.airbyte.cdk.message.DestinationMessage
|
||||
import io.airbyte.cdk.test.util.destination_process.DestinationProcessFactory
|
||||
import io.airbyte.cdk.load.command.DestinationCatalog
|
||||
import io.airbyte.cdk.load.command.DestinationStream
|
||||
import io.airbyte.cdk.load.message.DestinationMessage
|
||||
import io.airbyte.cdk.load.test.util.destination_process.DestinationProcessFactory
|
||||
import io.airbyte.protocol.models.v0.AirbyteMessage
|
||||
import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage
|
||||
import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage.AirbyteStreamStatus
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.test.util
|
||||
package io.airbyte.cdk.load.test.util
|
||||
|
||||
fun interface NameMapper {
|
||||
/**
|
||||
@@ -2,10 +2,10 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.test.util
|
||||
package io.airbyte.cdk.load.test.util
|
||||
|
||||
import io.airbyte.cdk.data.ObjectValue
|
||||
import io.airbyte.cdk.message.DestinationRecord.Change
|
||||
import io.airbyte.cdk.load.data.ObjectValue
|
||||
import io.airbyte.cdk.load.message.DestinationRecord.Change
|
||||
import java.time.Instant
|
||||
import java.util.UUID
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.test.util
|
||||
package io.airbyte.cdk.load.test.util
|
||||
|
||||
import io.airbyte.cdk.data.AirbyteValue
|
||||
import io.airbyte.cdk.data.IntegerValue
|
||||
import io.airbyte.cdk.data.NullValue
|
||||
import io.airbyte.cdk.data.ObjectValue
|
||||
import io.airbyte.cdk.load.data.AirbyteValue
|
||||
import io.airbyte.cdk.load.data.IntegerValue
|
||||
import io.airbyte.cdk.load.data.NullValue
|
||||
import io.airbyte.cdk.load.data.ObjectValue
|
||||
import kotlin.reflect.jvm.jvmName
|
||||
|
||||
class RecordDiffer(
|
||||
@@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
*/
|
||||
|
||||
package io.airbyte.cdk.test.util.destination_process
|
||||
package io.airbyte.cdk.load.test.util.destination_process
|
||||
|
||||
import io.airbyte.cdk.command.ConfigurationSpecification
|
||||
import io.airbyte.protocol.models.v0.AirbyteMessage
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user