diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index aab113da4a8..18346e5c79d 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 5.5.1 + dockerImageTag: 5.5.2 dockerRepository: airbyte/source-stripe documentationUrl: https://docs.airbyte.com/integrations/sources/stripe erdUrl: https://dbdocs.io/airbyteio/source-stripe?view=relationships diff --git a/airbyte-integrations/connectors/source-stripe/pyproject.toml b/airbyte-integrations/connectors/source-stripe/pyproject.toml index 48387a5d681..b19c3ad6e47 100644 --- a/airbyte-integrations/connectors/source-stripe/pyproject.toml +++ b/airbyte-integrations/connectors/source-stripe/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "5.5.1" +version = "5.5.2" name = "source-stripe" description = "Source implementation for Stripe." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index b71c52a6100..cae3f950913 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -14,6 +14,9 @@ import requests from airbyte_cdk import BackoffStrategy from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import ExponentialBackoffStrategy +from airbyte_cdk.sources.streams.checkpoint import Cursor +from airbyte_cdk.sources.streams.checkpoint.resumable_full_refresh_cursor import ResumableFullRefreshCursor +from airbyte_cdk.sources.streams.checkpoint.substream_resumable_full_refresh_cursor import SubstreamResumableFullRefreshCursor from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler @@ -208,6 +211,21 @@ class StripeStream(HttpStream, ABC): """ return 0 if IS_TESTING else super(StripeStream, self).retry_factor + def get_cursor(self) -> Optional[Cursor]: + """ + RFR is breaking the pagination in Stripe today. The stream is instantiated using the stream facade here. During the read, this goes + through the concurrent code here so that we can read full refresh streams concurrently. + + However, as there are no cursors and the read records is the one from the HttpStream, we end up assigning the + ResumableFullRefresCursor and hence only read a single page. + + In order to avoid that, we will assume there are no cursor if the cursor if RFR. + """ + parent_cursor = super().get_cursor() + if isinstance(parent_cursor, (ResumableFullRefreshCursor, SubstreamResumableFullRefreshCursor)): + return None + return parent_cursor + class IStreamSelector(ABC): @abstractmethod diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_accounts.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_accounts.py new file mode 100644 index 00000000000..db793ae0976 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_accounts.py @@ -0,0 +1,93 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from source_stripe import SourceStripe + +_STREAM_NAME = "accounts" +_ACCOUNT_ID = "acct_1G9HZLIEn49ers" +_CLIENT_SECRET = "ConfigBuilder default client secret" +_NOW = datetime.now(timezone.utc) +_CONFIG = { + "client_secret": _CLIENT_SECRET, + "account_id": _ACCOUNT_ID, +} +_NO_STATE = StateBuilder().build() +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _create_config() -> ConfigBuilder: + return ConfigBuilder().with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _create_catalog(sync_mode: SyncMode = SyncMode.full_refresh) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(name="accounts", sync_mode=sync_mode).build() + + +def _create_accounts_request() -> StripeRequestBuilder: + return StripeRequestBuilder.accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _create_response() -> HttpResponseBuilder: + return create_response_builder( + response_template=find_template(_STREAM_NAME, __file__), + records_path=FieldPath("data"), + pagination_strategy=StripePaginationStrategy(), + ) + + +def _create_record() -> RecordBuilder: + return create_record_builder( + find_template(_STREAM_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + ) + + +@freezegun.freeze_time(_NOW.isoformat()) +class AccountsTest(TestCase): + @HttpMocker() + def test_full_refresh(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record()).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert len(actual_messages.records) == 1 + + @HttpMocker() + def test_pagination(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record().with_id("last_record_id_from_first_page")).with_pagination().build(), + ) + http_mocker.get( + _create_accounts_request().with_limit(100).with_starting_after("last_record_id_from_first_page").build(), + _create_response().with_record(record=_create_record()).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert len(actual_messages.records) == 2 diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 82963621c51..c157fc3b181 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -235,36 +235,37 @@ Each record is marked with `is_deleted` flag when the appropriate event happens Expand to review | Version | Date | Pull Request | Subject | -| :------ | :--------- | :-------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 5.5.1 | 2024-08-10 | [43105](https://github.com/airbytehq/airbyte/pull/43105) | Update dependencies | -| 5.5.0 | 2024-08-08 | [43302](https://github.com/airbytehq/airbyte/pull/43302) | Fix problem with state not updating and upgrade cdk 4 -| 5.4.12 | 2024-07-31 | [41985](https://github.com/airbytehq/airbyte/pull/41985) | Expand Invoice discounts and tax rates -| 5.4.11 | 2024-07-27 | [42623](https://github.com/airbytehq/airbyte/pull/42623) | Update dependencies | -| 5.4.10 | 2024-07-20 | [42305](https://github.com/airbytehq/airbyte/pull/42305) | Update dependencies | -| 5.4.9 | 2024-07-13 | [41760](https://github.com/airbytehq/airbyte/pull/41760) | Update dependencies | -| 5.4.8 | 2024-07-10 | [41477](https://github.com/airbytehq/airbyte/pull/41477) | Update dependencies | -| 5.4.7 | 2024-07-09 | [40869](https://github.com/airbytehq/airbyte/pull/40869) | Update dependencies | -| 5.4.6 | 2024-07-08 | [41044](https://github.com/airbytehq/airbyte/pull/41044) | Use latest `CDK` version possible | -| 5.4.5 | 2024-06-25 | [40404](https://github.com/airbytehq/airbyte/pull/40404) | Update dependencies | -| 5.4.4 | 2024-06-22 | [40040](https://github.com/airbytehq/airbyte/pull/40040) | Update dependencies | -| 5.4.3 | 2024-06-06 | [39284](https://github.com/airbytehq/airbyte/pull/39284) | [autopull] Upgrade base image to v1.2.2 | -| 5.4.2 | 2024-06-11 | [39412](https://github.com/airbytehq/airbyte/pull/39412) | Removed `invoice.upcomming` event type from (incremental sync) for `Invoices` stream | -| 5.4.1 | 2024-06-11 | [39393](https://github.com/airbytehq/airbyte/pull/39393) | Added missing `event types` (incremental sync) for `Invoices` stream | -| 5.4.0 | 2024-06-05 | [39138](https://github.com/airbytehq/airbyte/pull/39138) | Fixed the `Refunds` stream missing data for the `incremental` sync | -| 5.3.9 | 2024-05-22 | [38550](https://github.com/airbytehq/airbyte/pull/38550) | Update authenticator package | -| 5.3.8 | 2024-05-15 | [38248](https://github.com/airbytehq/airbyte/pull/38248) | Replace AirbyteLogger with logging.Logger | -| 5.3.7 | 2024-04-24 | [36663](https://github.com/airbytehq/airbyte/pull/36663) | Schema descriptions | -| 5.3.6 | 2024-04-18 | [37448](https://github.com/airbytehq/airbyte/pull/37448) | Ensure AirbyteTracedException in concurrent CDK are emitted with the right type | -| 5.3.5 | 2024-04-18 | [37418](https://github.com/airbytehq/airbyte/pull/37418) | Ensure python return code != 0 in case of error | -| 5.3.4 | 2024-04-11 | [37406](https://github.com/airbytehq/airbyte/pull/37406) | Update CDK version to have partitioned state fix | -| 5.3.3 | 2024-04-11 | [37001](https://github.com/airbytehq/airbyte/pull/37001) | Update airbyte-cdk to flush print buffer for every message | -| 5.3.2 | 2024-04-11 | [36964](https://github.com/airbytehq/airbyte/pull/36964) | Update CDK version to fix breaking change before another devs work on it | -| 5.3.1 | 2024-04-10 | [36960](https://github.com/airbytehq/airbyte/pull/36960) | Remove unused imports | -| 5.3.0 | 2024-03-12 | [35978](https://github.com/airbytehq/airbyte/pull/35978) | Upgrade CDK to start emitting record counts with state and full refresh state | -| 5.2.4 | 2024-02-12 | [35137](https://github.com/airbytehq/airbyte/pull/35137) | Fix license in `pyproject.toml` | -| 5.2.3 | 2024-02-09 | [35068](https://github.com/airbytehq/airbyte/pull/35068) | Manage dependencies with Poetry. | -| 5.2.2 | 2024-01-31 | [34619](https://github.com/airbytehq/airbyte/pull/34619) | Events stream concurrent on incremental syncs | -| 5.2.1 | 2024-01-18 | [34495](https://github.com/airbytehq/airbyte/pull/34495) | Fix deadlock issue | +|:--------|:-----------| :-------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 5.5.2 | 2024-08-28 | [44862](https://github.com/airbytehq/airbyte/pull/44862) | Fix RFR pagination issue | +| 5.5.1 | 2024-08-10 | [43105](https://github.com/airbytehq/airbyte/pull/43105) | Update dependencies | +| 5.5.0 | 2024-08-08 | [43302](https://github.com/airbytehq/airbyte/pull/43302) | Fix problem with state not updating and upgrade cdk 4 +| 5.4.12 | 2024-07-31 | [41985](https://github.com/airbytehq/airbyte/pull/41985) | Expand Invoice discounts and tax rates +| 5.4.11 | 2024-07-27 | [42623](https://github.com/airbytehq/airbyte/pull/42623) | Update dependencies | +| 5.4.10 | 2024-07-20 | [42305](https://github.com/airbytehq/airbyte/pull/42305) | Update dependencies | +| 5.4.9 | 2024-07-13 | [41760](https://github.com/airbytehq/airbyte/pull/41760) | Update dependencies | +| 5.4.8 | 2024-07-10 | [41477](https://github.com/airbytehq/airbyte/pull/41477) | Update dependencies | +| 5.4.7 | 2024-07-09 | [40869](https://github.com/airbytehq/airbyte/pull/40869) | Update dependencies | +| 5.4.6 | 2024-07-08 | [41044](https://github.com/airbytehq/airbyte/pull/41044) | Use latest `CDK` version possible | +| 5.4.5 | 2024-06-25 | [40404](https://github.com/airbytehq/airbyte/pull/40404) | Update dependencies | +| 5.4.4 | 2024-06-22 | [40040](https://github.com/airbytehq/airbyte/pull/40040) | Update dependencies | +| 5.4.3 | 2024-06-06 | [39284](https://github.com/airbytehq/airbyte/pull/39284) | [autopull] Upgrade base image to v1.2.2 | +| 5.4.2 | 2024-06-11 | [39412](https://github.com/airbytehq/airbyte/pull/39412) | Removed `invoice.upcomming` event type from (incremental sync) for `Invoices` stream | +| 5.4.1 | 2024-06-11 | [39393](https://github.com/airbytehq/airbyte/pull/39393) | Added missing `event types` (incremental sync) for `Invoices` stream | +| 5.4.0 | 2024-06-05 | [39138](https://github.com/airbytehq/airbyte/pull/39138) | Fixed the `Refunds` stream missing data for the `incremental` sync | +| 5.3.9 | 2024-05-22 | [38550](https://github.com/airbytehq/airbyte/pull/38550) | Update authenticator package | +| 5.3.8 | 2024-05-15 | [38248](https://github.com/airbytehq/airbyte/pull/38248) | Replace AirbyteLogger with logging.Logger | +| 5.3.7 | 2024-04-24 | [36663](https://github.com/airbytehq/airbyte/pull/36663) | Schema descriptions | +| 5.3.6 | 2024-04-18 | [37448](https://github.com/airbytehq/airbyte/pull/37448) | Ensure AirbyteTracedException in concurrent CDK are emitted with the right type | +| 5.3.5 | 2024-04-18 | [37418](https://github.com/airbytehq/airbyte/pull/37418) | Ensure python return code != 0 in case of error | +| 5.3.4 | 2024-04-11 | [37406](https://github.com/airbytehq/airbyte/pull/37406) | Update CDK version to have partitioned state fix | +| 5.3.3 | 2024-04-11 | [37001](https://github.com/airbytehq/airbyte/pull/37001) | Update airbyte-cdk to flush print buffer for every message | +| 5.3.2 | 2024-04-11 | [36964](https://github.com/airbytehq/airbyte/pull/36964) | Update CDK version to fix breaking change before another devs work on it | +| 5.3.1 | 2024-04-10 | [36960](https://github.com/airbytehq/airbyte/pull/36960) | Remove unused imports | +| 5.3.0 | 2024-03-12 | [35978](https://github.com/airbytehq/airbyte/pull/35978) | Upgrade CDK to start emitting record counts with state and full refresh state | +| 5.2.4 | 2024-02-12 | [35137](https://github.com/airbytehq/airbyte/pull/35137) | Fix license in `pyproject.toml` | +| 5.2.3 | 2024-02-09 | [35068](https://github.com/airbytehq/airbyte/pull/35068) | Manage dependencies with Poetry. | +| 5.2.2 | 2024-01-31 | [34619](https://github.com/airbytehq/airbyte/pull/34619) | Events stream concurrent on incremental syncs | +| 5.2.1 | 2024-01-18 | [34495](https://github.com/airbytehq/airbyte/pull/34495) | Fix deadlock issue | | 5.2.0 | 2024-01-18 | [34347](https://github.com/airbytehq/airbyte/pull//34347) | Add new fields invoices and subscription streams. Upgrade the CDK for better memory usage. | | 5.1.3 | 2023-12-18 | [33306](https://github.com/airbytehq/airbyte/pull/33306/) | Adding integration tests | | 5.1.2 | 2024-01-04 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib |