jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix(source-google-ads): Fix custom query regex conditions (#69802)

Browse Source
This commit is contained in:
Anatolii Yatsuk
2025-11-24 16:37:11 +02:00
committed by GitHub
parent 24efb986e0
commit b85b63b0e8
5 changed files with 50 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
airbyte-integrations/connectors/source-google-ads/metadata.yaml
View File

@@ -11,7 +11,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: 253487c0-2246-43ba-a21f-5116b20a2c50
dockerImageTag: 4.1.0
dockerImageTag: 4.1.1
dockerRepository: airbyte/source-google-ads
documentationUrl: https://docs.airbyte.com/integrations/sources/google-ads
externalDocumentationUrls:
@@ -32,10 +32,8 @@ data:
registryOverrides:
cloud:
enabled: true
dockerImageTag: 4.0.2
oss:
enabled: true
dockerImageTag: 4.0.2
releaseStage: generally_available
releases:
rolloutConfiguration:

2
airbyte-integrations/connectors/source-google-ads/pyproject.toml
View File

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
build-backend = "poetry.core.masonry.api"
[tool.poetry]
version = "4.1.0"
version = "4.1.1"
name = "source-google-ads"
description = "Source implementation for Google Ads."
authors = [ "Airbyte <contact@airbyte.io>",]

42
airbyte-integrations/connectors/source-google-ads/source_google_ads/manifest.yaml
View File

@@ -1016,7 +1016,19 @@ definitions:
"datetime_format": "%Y-%m-%d"
}
}
condition: "{{ (components_values.get('query', '').count('segments.date') == 1 and (components_values.get('query') | regex_search('SELECT.*segments\\.date.*FROM')) is not none) or (components_values.get('query', '').count('segments.date') == 2 and (components_values.get('query') | regex_search('SELECT.*segments\\.date.*FROM')) is not none and (components_values.get('query') | regex_search('ORDER BY.*?segments\\.date.*?LIMIT')) is not none) }}"
condition: >-
{{
(
components_values.get('query', '').count('segments.date') == 1
and (components_values.get('query') | regex_search('(SELECT[\s\S]*?segments\\.date[\s\S]*?FROM)'))
)
or
(
components_values.get('query', '').count('segments.date') == 2
and (components_values.get('query') | regex_search('(SELECT[\s\S]*?segments\\.date[\s\S]*?FROM)'))
and (components_values.get('query') | regex_search('(ORDER BY[\s\S]*?segments\\.date)'))
)
}}
create_or_update: true
- type: ComponentMappingDefinition
field_path:
@@ -1033,7 +1045,19 @@ definitions:
- $parameters
- cursor_field
value: "segments.date"
condition: "{{ (components_values.get('query', '').count('segments.date') == 1 and (components_values.get('query') | regex_search('SELECT.*segments\\.date.*FROM')) is not none) or (components_values.get('query', '').count('segments.date') == 2 and (components_values.get('query') | regex_search('SELECT.*segments\\.date.*FROM')) is not none and (components_values.get('query') | regex_search('ORDER BY.*?segments\\.date.*?LIMIT')) is not none) }}"
condition: >-
{{
(
components_values.get('query', '').count('segments.date') == 1
and (components_values.get('query') | regex_search('(SELECT[\s\S]*?segments\\.date[\s\S]*?FROM)'))
)
or
(
components_values.get('query', '').count('segments.date') == 2
and (components_values.get('query') | regex_search('(SELECT[\s\S]*?segments\\.date[\s\S]*?FROM)'))
and (components_values.get('query') | regex_search('(ORDER BY[\s\S]*?segments\\.date)'))
)
}}
create_or_update: true
- type: ComponentMappingDefinition
field_path:
@@ -1047,7 +1071,19 @@ definitions:
- schema_loader
- cursor_field
value: "segments.date"
condition: "{{ (components_values.get('query', '').count('segments.date') == 1 and (components_values.get('query') | regex_search('SELECT.*segments\\.date.*FROM')) is not none) or (components_values.get('query', '').count('segments.date') == 2 and (components_values.get('query') | regex_search('SELECT.*segments\\.date.*FROM')) is not none and (components_values.get('query') | regex_search('ORDER BY.*?segments\\.date.*?LIMIT')) is not none) }}"
condition: >-
{{
(
components_values.get('query', '').count('segments.date') == 1
and (components_values.get('query') | regex_search('(SELECT[\s\S]*?segments\\.date[\s\S]*?FROM)'))
)
or
(
components_values.get('query', '').count('segments.date') == 2
and (components_values.get('query') | regex_search('(SELECT[\s\S]*?segments\\.date[\s\S]*?FROM)'))
and (components_values.get('query') | regex_search('(ORDER BY[\s\S]*?segments\\.date)'))
)
}}
create_or_update: true
- type: ComponentMappingDefinition
field_path:

10
airbyte-integrations/connectors/source-google-ads/unit_tests/test_empty_streams.py
View File

@@ -244,7 +244,9 @@ def test_custom_query_stream(customers, config_for_custom_query_tests, requests_
False,
),
("SELECT ad_group_ad.ad.name, segments.date FROM ad_group_ad", True),
("SELECT \n ad_group_ad.ad.name, \n segments.date FROM ad_group_ad ORDER BY \n segments.date DESC", True),
("SELECT ad_group_ad.ad.name, segments.date FROM ad_group_ad ORDER BY segments.date DESC", True),
("SELECT ad_group_ad.ad.name, segments.date FROM ad_group_ad ORDER BY ad_group_ad.ad.name, segments.date DESC", True),
("SELECT ad_group_ad.ad.name, segments.date FROM ad_group_ad ORDER BY segments.date DESC LIMIT 100", True),
(
"SELECT ad_group_ad.ad.name, segments.date FROM ad_group_ad ORDER BY ad_group_ad.ad.name DESC, segments.date DESC LIMIT 100",
@@ -254,8 +256,12 @@ def test_custom_query_stream(customers, config_for_custom_query_tests, requests_
"SELECT ad_group_ad.ad.name, segments.date FROM ad_group_ad ORDER BY segments.date DESC, ad_group_ad.ad.name DESC LIMIT 100",
True,
),
# This query currently gets incremental sync due to manifest regex condition matching SELECT.*segments.date.*FROM pattern
("SELECT ad_group_ad.ad.name, segments.date FROM ad_group_ad WHERE segments.date DURING LAST_30_DAYS", True),
("SELECT ad_group_ad.ad.name, segments.date FROM ad_group_ad WHERE segments.date DURING LAST_30_DAYS", False),
("SELECT ad_group_ad.ad.name, segments.date FROM ad_group_ad WHERE \n segments.date DURING LAST_30_DAYS", False),
(
"SELECT ad_group_ad.ad.name, segments.date FROM ad_group_ad WHERE segments.date DURING LAST_30_DAYS ORDER BY ad_group_ad.ad.name",
False,
),
],
)
def test_custom_query_stream_with_different_queries(query, expected_incremental_sync, config_for_custom_query_tests, requests_mock):