jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Source-MSSQL: Reduce Debezium polling frequency by configuring poll.interval.ms (#62491)

Browse Source
This commit is contained in:
Yarden Carmeli
2025-07-29 12:14:35 -07:00
committed by GitHub
parent c328800554
commit 89b66abe3a
3 changed files with 16 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
airbyte-integrations/connectors/source-mssql/metadata.yaml
View File

@@ -9,7 +9,7 @@ data:
connectorSubtype: database
connectorType: source
definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1
dockerImageTag: 4.2.3
dockerImageTag: 4.2.4
dockerRepository: airbyte/source-mssql
documentationUrl: https://docs.airbyte.com/integrations/sources/mssql
githubIssueLabel: source-mssql

13
airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java
View File

@@ -33,6 +33,11 @@ public class MssqlCdcHelper {
// Test execution latency is lower when heartbeats are more frequent.
private static final Duration HEARTBEAT_INTERVAL_IN_TESTS = Duration.ofSeconds(1L);
private static final Duration POLL_INTERVAL = Duration.ofSeconds(5L);
// The poll.interval.ms must be lower than the heartbeat.interval.ms
private static final Duration POLL_INTERVAL_IN_TESTS = Duration.ofMillis(500L);
public enum ReplicationMethod {
STANDARD,
CDC
@@ -104,6 +109,14 @@ public class MssqlCdcHelper {
: HEARTBEAT_INTERVAL;
props.setProperty("heartbeat.interval.ms", Long.toString(heartbeatInterval.toMillis()));
// Set poll.interval.ms to 5s. This parameter will determine how long Debezium will wait before
// querying for new data. It must be lower than heartbeat.interval.ms
final Duration pollInterval =
(database.getSourceConfig().has("is_test") && database.getSourceConfig().get("is_test").asBoolean())
? POLL_INTERVAL_IN_TESTS
: POLL_INTERVAL;
props.setProperty("poll.interval.ms", Long.toString(pollInterval.toMillis()));
if (config.has("ssl_method")) {
final JsonNode sslConfig = config.get("ssl_method");
final String sslMethod = sslConfig.get("ssl_method").asText();

3
docs/integrations/sources/mssql.md
View File

@@ -445,8 +445,9 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura
| Version | Date | Pull Request | Subject |
|:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------|
| 4.2.4 | 2025-07-03 | [62491](https://github.com/airbytehq/airbyte/pull/62491) | Improve Debezium performance by configuring the poll interval parameter. |
| 4.2.3 | 2025-07-01 | [62052](https://github.com/airbytehq/airbyte/pull/62052) | Revert change to CDK interface signature. |
| 4.2.2 | 2025-06-25 | [61729](https://github.com/airbytehq/airbyte/pull/61729) | Support the use of logical primary keys for CDC. |
| 4.2.2 | 2025-06-25 | [61729](https://github.com/airbytehq/airbyte/pull/61729) | Support the use of logical primary keys for CDC. |
| 4.2.1 | 2025-06-23 | [62015](https://github.com/airbytehq/airbyte/pull/62015) | Fix previous merge. Improve cutoff date handling |
| 4.2.0 | 2025-06-19 | [62015](https://github.com/airbytehq/airbyte/pull/61685) | Add an option to exclude today's data from cursor based incremental syncs when using temporal cursor (datetime). |
| 4.1.29 | 2025-06-03 | [61320](https://github.com/airbytehq/airbyte/pull/61320) | Add error handling for connection issues and adopt the latest CDK version. |