jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2026-01-05 12:05:28 -05:00
Code Issues Packages Projects Releases Wiki Activity

Populate the PK from the Singer discovery run (#2713) (#4789)

Browse Source 
When running Singer discovery, use the `key_properties` field to populate the `source_defined_primary_key` stream meta.
This commit is contained in:
Artjoms Iskovs
2021-08-09 18:11:22 +01:00
committed by GitHub
parent 219877f778
commit f790fee57c
6 changed files with 194 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
airbyte-cdk/python/airbyte_cdk/sources/singer/source.py
View File

@@ -100,7 +100,9 @@ class SingerSource(Source):
def _discover_internal(self, logger: AirbyteLogger, config_path: str) -> Catalogs:
cmd = self.discover_cmd(logger, config_path)
catalogs = SingerHelper.get_catalogs(logger, cmd, self.get_sync_mode_overrides(), self.get_excluded_streams())
catalogs = SingerHelper.get_catalogs(
logger, cmd, self.get_sync_mode_overrides(), self.get_primary_key_overrides(), self.get_excluded_streams()
)
return catalogs
def check(self, logger: AirbyteLogger, config_container: ConfigContainer) -> AirbyteConnectionStatus:
@@ -147,6 +149,14 @@ class SingerSource(Source):
"""
return {}
def get_primary_key_overrides(self) -> Dict[str, List[str]]:
"""
Similar to get_sync_mode_overrides but for primary keys.
:return: A dict from stream name to the list of primary key fields for the stream.
"""
return {}
def get_excluded_streams(self) -> List[str]:
"""
This method provide ability to exclude some streams from catalog