a013fad5a9
* [ISSUE #26581] per partition cursor * [ISSUE #26581] format * [ISSUE #26581] clean up state management * [ISSUE #26581] improving Hashabledict * [ISSUE #26581] format cdk * [ISSUE #26581] fix tests * [ISSUE #26581] code review from girarda * Retrigger pipeline * Decouple cursor and stream slicer and pushing state management as far up cursor as possible * Format cdk * Small fixes/comments * DatetimeBasedCursor should not update state based on slice (for now at least since it wasn't doing this before) * [ISSUE #26581] code review * Automated Commit - Formatting Changes * [ISSUE #26581] validation overlapping keys * [ISSUE #26581] add typing * [ISSUE #26581] code review * Remove SyncMode from stream_slices * Removing SyncMode from stream_slices up until SimpleRetriever and fixing typing * [ISSUE-26434] replacing Record primitive by class * [ISSUE-26434] update Cursor.update_state to use new record object * Issue 26343/data feed incremental sync solution 2 (#27481) * TMP [ISSUE-26434] first solution to enable stop condition on pagination * TMP [ISSUE-26434] second solution to enable stop condition on pagination * TMP [ISSUE-26434] second solution fix * [ISSUE #26343] fixing behavior and adding tests * [ISSUE #26343] only updating state once a slice to allow for data feed * [ISSUE #26343] removing freezing of cursor * format cdk * [ISSUE #26343] ensure data_feed doesn't have end_datetime * [ISSUE #26343] self review * [ISSUE #26343] code review * [ISSUE #26343] code review clean up * [ISSUE #26343] code review clean up * Code review * [ISSUE #26343] add warn log message in DatetimeBasedCursor * format * Format
65 lines
2.2 KiB
Python
65 lines
2.2 KiB
Python
#
|
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
#
|
|
|
|
import pytest
|
|
from airbyte_cdk.models import (
|
|
AirbyteControlConnectorConfigMessage,
|
|
AirbyteControlMessage,
|
|
AirbyteMessage,
|
|
AirbyteStateMessage,
|
|
OrchestratorType,
|
|
Type,
|
|
)
|
|
from airbyte_cdk.sources.message import InMemoryMessageRepository
|
|
|
|
A_CONTROL = AirbyteControlMessage(
|
|
type=OrchestratorType.CONNECTOR_CONFIG,
|
|
emitted_at=0,
|
|
connectorConfig=AirbyteControlConnectorConfigMessage(config={"a config": "value"}),
|
|
)
|
|
ANOTHER_CONTROL = AirbyteControlMessage(
|
|
type=OrchestratorType.CONNECTOR_CONFIG,
|
|
emitted_at=0,
|
|
connectorConfig=AirbyteControlConnectorConfigMessage(config={"another config": "another value"}),
|
|
)
|
|
|
|
|
|
def test_given_no_messages_when_consume_queue_then_return_empty():
|
|
repo = InMemoryMessageRepository()
|
|
messages = list(repo.consume_queue())
|
|
assert messages == []
|
|
|
|
|
|
def test_given_messages_when_consume_queue_then_return_messages():
|
|
repo = InMemoryMessageRepository()
|
|
first_message = AirbyteMessage(type=Type.CONTROL, control=A_CONTROL)
|
|
repo.emit_message(first_message)
|
|
second_message = AirbyteMessage(type=Type.CONTROL, control=ANOTHER_CONTROL)
|
|
repo.emit_message(second_message)
|
|
|
|
messages = repo.consume_queue()
|
|
|
|
assert list(messages) == [first_message, second_message]
|
|
|
|
|
|
def test_given_message_is_consumed_when_consume_queue_then_remove_message_from_queue():
|
|
repo = InMemoryMessageRepository()
|
|
first_message = AirbyteMessage(type=Type.CONTROL, control=A_CONTROL)
|
|
repo.emit_message(first_message)
|
|
second_message = AirbyteMessage(type=Type.CONTROL, control=ANOTHER_CONTROL)
|
|
repo.emit_message(second_message)
|
|
|
|
message_generator = repo.consume_queue()
|
|
consumed_message = next(message_generator)
|
|
assert consumed_message == first_message
|
|
|
|
second_message_generator = repo.consume_queue()
|
|
assert list(second_message_generator) == [second_message]
|
|
|
|
|
|
def test_given_message_is_not_control_nor_log_message_when_emit_message_then_raise_error():
|
|
repo = InMemoryMessageRepository()
|
|
with pytest.raises(ValueError):
|
|
repo.emit_message(AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": "state value"})))
|