jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2026-01-06 06:04:16 -05:00
Code Issues Packages Projects Releases Wiki Activity

Publish stream status messages in CDK (#24994)

Browse Source 
* Publish stream status messages in CDK

* Automated Commit - Formatting Changes

* Convert to StreamDescriptor

* Automated Commit - Formatting Changes

* Bump to latest protocol model

* Automated Commit - Formatting Changes

* Bump protocol version

* Add tests for stream status message creation

* Formatting

* Formatting

* Fix failing test

* Actually emit state message

* Automated Commit - Formatting Changes

* Bump airbyte-protocol

* PR feedback

* Fix parameter input

* Correctly yield status message

* PR feedback

* Formatting

* Fix failing tests

* Automated Commit - Formatting Changes

* Revert accidental change

* Automated Change

* Replace STOPPED with COMPLETE/INCOMPLETE

* Update source-facebook-marketing changelog

* Revert "Update source-facebook-marketing changelog"

This reverts commit 709edb800c.

---------

Co-authored-by: jdpgrailsdev <jdpgrailsdev@users.noreply.github.com>
This commit is contained in:
Jonathan Pearlin
2023-04-26 10:30:36 -04:00
committed by GitHub
parent 3e6c8a37b0
commit 2ebfa459cf
9 changed files with 242 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py
View File

@@ -13,6 +13,7 @@ from airbyte_cdk.models import (
AirbyteLogMessage,
AirbyteMessage,
AirbyteStateMessage,
AirbyteStreamStatus,
ConfiguredAirbyteCatalog,
ConfiguredAirbyteStream,
Level,
@@ -28,6 +29,7 @@ from airbyte_cdk.sources.streams.http.http import HttpStream
from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, split_config
from airbyte_cdk.utils.event_timing import create_timer
from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
@@ -113,6 +115,8 @@ class AbstractSource(Source, ABC):
continue
try:
timer.start_event(f"Syncing stream {configured_stream.stream.name}")
logger.info(f"Marking stream {configured_stream.stream.name} as STARTED")
yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.STARTED)
yield from self._read_stream(
logger=logger,
stream_instance=stream_instance,
@@ -120,10 +124,15 @@ class AbstractSource(Source, ABC):
state_manager=state_manager,
internal_config=internal_config,
)
logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED")
yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.COMPLETE)
except AirbyteTracedException as e:
yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE)
raise e
except Exception as e:
logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}")
logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED")
yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE)
display_message = stream_instance.get_error_display_message(e)
if display_message:
raise AirbyteTracedException.from_exception(e, message=display_message) from e
@@ -185,6 +194,10 @@ class AbstractSource(Source, ABC):
for record in record_iterator:
if record.type == MessageType.RECORD:
record_counter += 1
if record_counter == 1:
logger.info(f"Marking stream {stream_name} as RUNNING")
# If we just read the first record of the stream, emit the transition to the RUNNING state
yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.RUNNING)
yield record
logger.info(f"Read {record_counter} records from {stream_name} stream")