Add error handling and bump CDK version (#61320)
This commit is contained in:
Yarden Carmeli
@@ -5,7 +5,7 @@ plugins {
|
||||
}
|
||||
|
||||
airbyteJavaConnector {
|
||||
cdkVersionRequired = '0.48.8'
|
||||
cdkVersionRequired = '0.48.14'
|
||||
features = ['db-sources']
|
||||
useLocalCdk = false
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ data:
|
||||
connectorSubtype: database
|
||||
connectorType: source
|
||||
definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1
|
||||
dockerImageTag: 4.1.28
|
||||
dockerImageTag: 4.1.29
|
||||
dockerRepository: airbyte/source-mssql
|
||||
documentationUrl: https://docs.airbyte.com/integrations/sources/mssql
|
||||
githubIssueLabel: source-mssql
|
||||
|
||||
@@ -80,6 +80,7 @@ public class MssqlDebeziumStateUtil implements DebeziumStateUtil {
|
||||
// There is no need to construct an initial state after it was already constructed in this run
|
||||
// Starting and stopping mssql debezium too many times causes it to hang during shutdown
|
||||
if (initialState.get() == null) {
|
||||
LOGGER.info("No initial state was found. Running Debezium state initialization...");
|
||||
properties.setProperty("heartbeat.interval.ms", "0");
|
||||
final JsonNode highWaterMark = constructLsnSnapshotState(database, database.getSourceConfig().get(JdbcUtils.DATABASE_KEY).asText());
|
||||
final AirbyteFileOffsetBackingStore emptyOffsetManager = AirbyteFileOffsetBackingStore.initializeState(null,
|
||||
|
||||
@@ -11,23 +11,81 @@ import io.airbyte.cdk.integrations.util.FailureType
|
||||
class MSSqlSourceExceptionHandler : ConnectorExceptionHandler() {
|
||||
override fun initializeErrorDictionary() {
|
||||
|
||||
val DATABASE_READ_ERROR = "Encountered an error while reading the database, will retry"
|
||||
|
||||
// include common error profiles
|
||||
super.initializeErrorDictionary()
|
||||
|
||||
// adding connector specific error profiles
|
||||
add(
|
||||
ConnectorErrorProfile(
|
||||
errorClass = "MS SQL Exception",
|
||||
errorClass = "MSSQL Exception",
|
||||
regexMatchingPattern =
|
||||
".*returned an incomplete response. The connection has been closed.*",
|
||||
"(?i).*returned an incomplete response. The connection has been closed.*",
|
||||
failureType = FailureType.TRANSIENT,
|
||||
externalMessage = DATABASE_READ_ERROR,
|
||||
externalMessage =
|
||||
"(?i).*returned an incomplete response. The connection has been closed.*",
|
||||
sampleInternalMessage =
|
||||
"com.microsoft.sqlserver.jdbc.SQLServerException: SQL Server returned an incomplete response. The connection has been closed.",
|
||||
referenceLinks = listOf("https://github.com/airbytehq/oncall/issues/6623")
|
||||
)
|
||||
),
|
||||
)
|
||||
add(
|
||||
ConnectorErrorProfile(
|
||||
errorClass = "MSSQL Exception",
|
||||
regexMatchingPattern =
|
||||
"(?i).*SQL Server did not return a response. The connection has been closed.*",
|
||||
failureType = FailureType.TRANSIENT,
|
||||
externalMessage =
|
||||
"Encountered an error while reading from the database, will retry",
|
||||
sampleInternalMessage =
|
||||
"com.microsoft.sqlserver.jdbc.SQLServerException: SQL Server did not return a response. The connection has been closed.",
|
||||
referenceLinks = listOf("https://github.com/airbytehq/oncall/issues/7757")
|
||||
),
|
||||
)
|
||||
add(
|
||||
ConnectorErrorProfile(
|
||||
errorClass = "MSSQL Exception",
|
||||
regexMatchingPattern = "(?i).*The connection is closed.*",
|
||||
failureType = FailureType.TRANSIENT,
|
||||
externalMessage = "The SQL Server connection was unexpectedly closed, will retry.",
|
||||
sampleInternalMessage =
|
||||
"com.microsoft.sqlserver.jdbc.SQLServerException: The connection is closed.",
|
||||
referenceLinks = listOf("https://github.com/airbytehq/oncall/issues/6438")
|
||||
),
|
||||
)
|
||||
add(
|
||||
// Error 1205
|
||||
// https://learn.microsoft.com/en-us/sql/relational-databases/errors-events/mssqlserver-1205-database-engine-error
|
||||
ConnectorErrorProfile(
|
||||
errorClass = "MSSQL Exception",
|
||||
regexMatchingPattern =
|
||||
"(?i).*was deadlocked on lock resources with another process and has been chosen as the deadlock victim. Rerun the transaction.*",
|
||||
failureType = FailureType.TRANSIENT,
|
||||
externalMessage =
|
||||
"Transaction conflicted with another process and was terminated, will retry.",
|
||||
sampleInternalMessage =
|
||||
"com.microsoft.sqlserver.jdbc.SQLServerException: " +
|
||||
"Transaction (Process ID 63) was deadlocked on lock resources with another process and has been chosen as the deadlock victim. Rerun the transaction.",
|
||||
referenceLinks = listOf("https://github.com/airbytehq/oncall/issues/6287")
|
||||
),
|
||||
)
|
||||
// This error occurs when Debezium encounters an exception.
|
||||
// We classify it as TRANSIENT since it may be resolved through automatic retries but can
|
||||
// also require investigation and manual intervention.
|
||||
add(
|
||||
ConnectorErrorProfile(
|
||||
errorClass = "Connect Exception",
|
||||
regexMatchingPattern = "(?i).*exception occurred in the change event producer.*",
|
||||
failureType = FailureType.TRANSIENT,
|
||||
externalMessage =
|
||||
"The sync encountered an unexpected error in the change event producer and has stopped. Please check the logs for details and troubleshoot accordingly.",
|
||||
sampleInternalMessage =
|
||||
"java.lang.RuntimeException: org.apache.kafka.connect.errors.ConnectException: " +
|
||||
"An exception occurred in the change event producer. This connector will be stopped.",
|
||||
referenceLinks =
|
||||
listOf(
|
||||
"https://docs.oracle.com/javase/9/docs/api/java/lang/RuntimeException.html"
|
||||
)
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -445,6 +445,7 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura
|
||||
|
||||
| Version | Date | Pull Request | Subject |
|
||||
|:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| 4.1.29 | 2025-06-03 | [61320](https://github.com/airbytehq/airbyte/pull/61320) | Add error handling for connection issues and adopt the latest CDK version. |
|
||||
| 4.1.28 | 2025-05-15 | [60311](https://github.com/airbytehq/airbyte/pull/60311) | Migrate to new gradle flow. |
|
||||
| 4.1.27 | 2025-04-28 | [59124](https://github.com/airbytehq/airbyte/pull/59124) | Fix _ab_cdc_event_serial_no datatype in addMetaDataToRowsFetchedOutsideDebezium method |
|
||||
| 4.1.26 | 2025-03-27 | [56401](https://github.com/airbytehq/airbyte/pull/56401) | Fix non-unique value clustered index issue |
|
||||
|
||||
Reference in New Issue
Block a user