jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

🐛 Source stripe: fix case where stream wont have a state attribute and needs to resolve get_updated_state and upgrade CDK 4 (remove availability strategy) (#43302)

Browse Source
This commit is contained in:
Aldo Gonzalez
2024-08-08 10:06:50 -06:00
committed by GitHub
parent 64c3870c96
commit 33eddee08f
28 changed files with 359 additions and 1260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
airbyte-integrations/connectors/source-stripe/metadata.yaml
View File

@@ -10,7 +10,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: e094cb9a-26de-4645-8761-65c0c425d1de
dockerImageTag: 5.4.12
dockerImageTag: 5.5.0
dockerRepository: airbyte/source-stripe
documentationUrl: https://docs.airbyte.com/integrations/sources/stripe
githubIssueLabel: source-stripe
@@ -25,7 +25,6 @@ data:
registries:
cloud:
enabled: true
dockerImageTag: 5.4.5
oss:
enabled: true
releaseStage: generally_available

62
airbyte-integrations/connectors/source-stripe/poetry.lock generated
View File

@@ -1,14 +1,14 @@
# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
[[package]]
name = "airbyte-cdk"
version = "2.4.0"
version = "4.3.2"
description = "A framework for writing Airbyte Connectors."
optional = false
python-versions = "<4.0,>=3.9"
python-versions = "<4.0,>=3.10"
files = [
{file = "airbyte_cdk-2.4.0-py3-none-any.whl", hash = "sha256:39470b2fe97f28959fcecb839d3080a8aba4a64a29dddf54a39f11f93f9f9ef7"},
{file = "airbyte_cdk-2.4.0.tar.gz", hash = "sha256:f973d2e17a6dd0416c4395139e2761a10b38aafa61e097eaacffebbe6164ef45"},
{file = "airbyte_cdk-4.3.2-py3-none-any.whl", hash = "sha256:1dd92de77e2212b13ed6f1e9c4b2559baa905eff6db5bf14b0fbaf1149e60e4c"},
{file = "airbyte_cdk-4.3.2.tar.gz", hash = "sha256:89db68167e214e5b55d5ef5f821f017ee64bf859f44c60bc0169071fa376e9af"},
]
[package.dependencies]
@@ -77,22 +77,22 @@ files = [
[[package]]
name = "attrs"
version = "23.2.0"
version = "24.2.0"
description = "Classes Without Boilerplate"
optional = false
python-versions = ">=3.7"
files = [
{file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"},
{file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"},
{file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"},
{file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"},
]
[package.extras]
cov = ["attrs[tests]", "coverage[toml] (>=5.3)"]
dev = ["attrs[tests]", "pre-commit"]
docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"]
tests = ["attrs[tests-no-zope]", "zope-interface"]
tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"]
tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"]
benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"]
tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"]
[[package]]
name = "backoff"
@@ -107,13 +107,13 @@ files = [
[[package]]
name = "bracex"
version = "2.4"
version = "2.5"
description = "Bash style brace expander."
optional = false
python-versions = ">=3.8"
files = [
{file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"},
{file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"},
{file = "bracex-2.5-py3-none-any.whl", hash = "sha256:d2fcf4b606a82ac325471affe1706dd9bbaa3536c91ef86a31f6b766f3dad1d0"},
{file = "bracex-2.5.tar.gz", hash = "sha256:0725da5045e8d37ea9592ab3614d8b561e22c3c5fde3964699be672e072ab611"},
]
[[package]]
@@ -591,13 +591,13 @@ extended-testing = ["jinja2 (>=3,<4)"]
[[package]]
name = "langsmith"
version = "0.1.93"
version = "0.1.98"
description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
optional = false
python-versions = "<4.0,>=3.8.1"
files = [
{file = "langsmith-0.1.93-py3-none-any.whl", hash = "sha256:811210b9d5f108f36431bd7b997eb9476a9ecf5a2abd7ddbb606c1cdcf0f43ce"},
{file = "langsmith-0.1.93.tar.gz", hash = "sha256:285b6ad3a54f50fa8eb97b5f600acc57d0e37e139dd8cf2111a117d0435ba9b4"},
{file = "langsmith-0.1.98-py3-none-any.whl", hash = "sha256:f79e8a128652bbcee4606d10acb6236973b5cd7dde76e3741186d3b97b5698e9"},
{file = "langsmith-0.1.98.tar.gz", hash = "sha256:e07678219a0502e8f26d35294e72127a39d25e32fafd091af5a7bb661e9a6bd1"},
]
[package.dependencies]
@@ -954,19 +954,19 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
[[package]]
name = "pyjwt"
version = "2.8.0"
version = "2.9.0"
description = "JSON Web Token implementation in Python"
optional = false
python-versions = ">=3.7"
python-versions = ">=3.8"
files = [
{file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"},
{file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},
{file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"},
{file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"},
]
[package.extras]
crypto = ["cryptography (>=3.4.0)"]
dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"]
docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"]
dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"]
docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"]
tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"]
[[package]]
@@ -1232,13 +1232,13 @@ fixture = ["fixtures"]
[[package]]
name = "setuptools"
version = "71.1.0"
version = "72.1.0"
description = "Easily download, build, install, upgrade, and uninstall Python packages"
optional = false
python-versions = ">=3.8"
files = [
{file = "setuptools-71.1.0-py3-none-any.whl", hash = "sha256:33874fdc59b3188304b2e7c80d9029097ea31627180896fb549c578ceb8a0855"},
{file = "setuptools-71.1.0.tar.gz", hash = "sha256:032d42ee9fb536e33087fb66cac5f840eb9391ed05637b3f2a76a7c8fb477936"},
{file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"},
{file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"},
]
[package.extras]
@@ -1434,5 +1434,5 @@ files = [
[metadata]
lock-version = "2.0"
python-versions = "^3.9,<3.12"
content-hash = "168a51efb700c60317faaa0511534aad791b8ba2bd8aebb3ad102333eb3e30b4"
python-versions = "^3.10,<3.12"
content-hash = "8962d733e3e6727e34c89168bdda605394383945fe8ecf98310e64277df9fcc3"

6
airbyte-integrations/connectors/source-stripe/pyproject.toml
View File

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
build-backend = "poetry.core.masonry.api"
[tool.poetry]
version = "5.4.12"
version = "5.5.0"
name = "source-stripe"
description = "Source implementation for Stripe."
authors = [ "Airbyte <contact@airbyte.io>",]
@@ -16,10 +16,10 @@ repository = "https://github.com/airbytehq/airbyte"
include = "source_stripe"
[tool.poetry.dependencies]
python = "^3.9,<3.12"
python = "^3.10,<3.12"
stripe = "==2.56.0"
pendulum = "==2.1.2"
airbyte-cdk = "^2"
airbyte-cdk = "^4"
[tool.poetry.scripts]
source-stripe = "source_stripe.run:run"

114
airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py
View File

@@ -1,114 +0,0 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#
import logging
from typing import Any, Mapping, Optional, Tuple
from airbyte_cdk.models import SyncMode
from airbyte_cdk.sources import Source
from airbyte_cdk.sources.streams import Stream
from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
from requests import HTTPError
from .stream_helpers import get_first_record_for_slice, get_first_stream_slice
STRIPE_ERROR_CODES = {
"more_permissions_required": "This is most likely due to insufficient permissions on the credentials in use. "
"Try to grant required permissions/scopes or re-authenticate",
"account_invalid": "The card, or account the card is connected to, is invalid. You need to contact your card issuer "
"to check that the card is working correctly.",
"oauth_not_supported": "Please use a different authentication method.",
}
class StripeAvailabilityStrategy(HttpAvailabilityStrategy):
def _check_availability_for_sync_mode(
self,
stream: Stream,
sync_mode: SyncMode,
logger: logging.Logger,
source: Optional["Source"],
stream_state: Optional[Mapping[str, Any]],
) -> Tuple[bool, Optional[str]]:
try:
# Some streams need a stream slice to read records (e.g. if they have a SubstreamPartitionRouter)
# Streams that don't need a stream slice will return `None` as their first stream slice.
stream_slice = get_first_stream_slice(stream, sync_mode, stream_state)
except StopIteration:
# If stream_slices has no `next()` item (Note - this is different from stream_slices returning [None]!)
# This can happen when a substream's `stream_slices` method does a `for record in parent_records: yield <something>`
# without accounting for the case in which the parent stream is empty.
reason = f"Cannot attempt to connect to stream {stream.name} - no stream slices were found, likely because the parent stream is empty."
return False, reason
except HTTPError as error:
is_available, reason = self.handle_http_error(stream, logger, source, error)
if not is_available:
reason = f"Unable to get slices for {stream.name} stream, because of error in parent stream. {reason}"
return is_available, reason
try:
get_first_record_for_slice(stream, sync_mode, stream_slice, stream_state)
return True, None
except StopIteration:
logger.info(f"Successfully connected to stream {stream.name}, but got 0 records.")
return True, None
except HTTPError as error:
is_available, reason = self.handle_http_error(stream, logger, source, error)
if not is_available:
reason = f"Unable to read {stream.name} stream. {reason}"
return is_available, reason
def check_availability(self, stream: Stream, logger: logging.Logger, source: Optional["Source"]) -> Tuple[bool, Optional[str]]:
"""
Check stream availability by attempting to read the first record of the
stream.
:param stream: stream
:param logger: source logger
:param source: (optional) source
:return: A tuple of (boolean, str). If boolean is true, then the stream
is available, and no str is required. Otherwise, the stream is unavailable
for some reason and the str should describe what went wrong and how to
resolve the unavailability, if possible.
"""
is_available, reason = self._check_availability_for_sync_mode(stream, SyncMode.full_refresh, logger, source, None)
if not is_available or not stream.supports_incremental:
return is_available, reason
return self._check_availability_for_sync_mode(stream, SyncMode.incremental, logger, source, {stream.cursor_field: 0})
def handle_http_error(
self, stream: Stream, logger: logging.Logger, source: Optional["Source"], error: HTTPError
) -> Tuple[bool, Optional[str]]:
status_code = error.response.status_code
if status_code not in [400, 403]:
raise error
parsed_error = error.response.json()
error_code = parsed_error.get("error", {}).get("code")
error_message = STRIPE_ERROR_CODES.get(error_code, parsed_error.get("error", {}).get("message"))
if not error_message:
raise error
doc_ref = self._visit_docs_message(logger, source)
reason = f"The endpoint {error.response.url} returned {status_code}: {error.response.reason}. {error_message}. {doc_ref} "
response_error_message = stream.parse_response_error_message(error.response)
if response_error_message:
reason += response_error_message
return False, reason
class StripeSubStreamAvailabilityStrategy(StripeAvailabilityStrategy):
def check_availability(self, stream: Stream, logger: logging.Logger, source: Optional[Source]) -> Tuple[bool, Optional[str]]:
"""Traverse through all the parents of a given stream and run availability strategy on each of them"""
try:
current_stream, parent_stream = stream, getattr(stream, "parent")
except AttributeError:
return super().check_availability(stream, logger, source)
if parent_stream:
parent_stream_instance = getattr(current_stream, "parent")
# Accessing the `availability_strategy` property will instantiate AvailabilityStrategy under the hood
availability_strategy = parent_stream_instance.availability_strategy
if availability_strategy:
is_available, reason = availability_strategy.check_availability(parent_stream_instance, logger, source)
if not is_available:
return is_available, reason
return super().check_availability(stream, logger, source)

4
airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
from .parent_incremental_stripe_sub_stream_error_handler import ParentIncrementalStripeSubStreamErrorHandler
from .stripe_error_handler import StripeErrorHandler
__all__ = ['StripeErrorHandler', 'ParentIncrementalStripeSubStreamErrorHandler']

23
airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/parent_incremental_stripe_sub_stream_error_handler.py Normal file
View File

@@ -0,0 +1,23 @@
#
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
#
from typing import Optional, Union
import requests
from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution
from source_stripe.error_handlers.stripe_error_handler import StripeErrorHandler
class ParentIncrementalStripeSubStreamErrorHandler(StripeErrorHandler):
def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
if not isinstance(response_or_exception, Exception) and response_or_exception.status_code == requests.codes.not_found:
# When running incremental sync with state, the returned parent object very likely will not contain sub-items
# as the events API does not support expandable items. Parent class will try getting sub-items from this object,
# then from its own API. In case there are no sub-items at all for this entity, API will raise 404 error.
self._logger.warning(
f"Data was not found for URL: {response_or_exception.request.url}. "
"If this is a path for getting child attributes like /v1/checkout/sessions/<session_id>/line_items when running "
"the incremental sync, you may safely ignore this warning."
)
return super().interpret_response(response_or_exception)

42
airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py Normal file
View File

@@ -0,0 +1,42 @@
#
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
#
import logging
from typing import Optional, Union
import requests
from airbyte_cdk.models import FailureType
from airbyte_cdk.sources.streams.http import HttpStream
from airbyte_cdk.sources.streams.http.error_handlers import HttpStatusErrorHandler
from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction
STRIPE_ERROR_CODES = {
"more_permissions_required": "This is most likely due to insufficient permissions on the credentials in use. "
"Try to grant required permissions/scopes or re-authenticate",
"account_invalid": "The card, or account the card is connected to, is invalid. You need to contact your card issuer "
"to check that the card is working correctly.",
"oauth_not_supported": "Please use a different authentication method.",
}
DOCS_URL = f"https://docs.airbyte.com/integrations/sources/stripe"
DOCUMENTATION_MESSAGE = f"Please visit {DOCS_URL} to learn more. "
class StripeErrorHandler(HttpStatusErrorHandler):
def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
if not isinstance(response_or_exception, Exception) and response_or_exception.status_code in (
requests.codes.bad_request,
requests.codes.forbidden,
):
parsed_error = response_or_exception.json()
error_code = parsed_error.get("error", {}).get("code")
error_message = STRIPE_ERROR_CODES.get(error_code, parsed_error.get("error", {}).get("message"))
if error_message:
reason = f"The endpoint {response_or_exception.url} returned {response_or_exception.status_code}: {response_or_exception.reason}. {error_message}. {DOCUMENTATION_MESSAGE} "
response_error_message = HttpStream.parse_response_error_message(response_or_exception)
if response_error_message:
reason += response_error_message
return ErrorResolution(response_action=ResponseAction.IGNORE, error_message=reason)
return super().interpret_response(response_or_exception)

3
airbyte-integrations/connectors/source-stripe/source_stripe/error_mappings/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
from .parent_incremental_stripe_sub_stream_error_mapping import PARENT_INCREMENTAL_STRIPE_SUB_STREAM_ERROR_MAPPING
__all__ = ['PARENT_INCREMENTAL_STRIPE_SUB_STREAM_ERROR_MAPPING']

13
airbyte-integrations/connectors/source-stripe/source_stripe/error_mappings/parent_incremental_stripe_sub_stream_error_mapping.py Normal file
View File

@@ -0,0 +1,13 @@
#
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
#
from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING
from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction
PARENT_INCREMENTAL_STRIPE_SUB_STREAM_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | {
404: ErrorResolution(
response_action=ResponseAction.IGNORE,
error_message="Data was not found for URL.",
),
}

41
airbyte-integrations/connectors/source-stripe/source_stripe/stream_helpers.py
View File

@@ -1,41 +0,0 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#
from typing import Any, Mapping, Optional
from airbyte_cdk.models import SyncMode
from airbyte_cdk.sources.streams.core import Stream, StreamData
def get_first_stream_slice(stream, sync_mode, stream_state) -> Optional[Mapping[str, Any]]:
"""
Gets the first stream_slice from a given stream's stream_slices.
:param stream: stream
:param sync_mode: sync_mode
:param stream_state: stream_state
:raises StopIteration: if there is no first slice to return (the stream_slices generator is empty)
:return: first stream slice from 'stream_slices' generator (`None` is a valid stream slice)
"""
# We wrap the return output of stream_slices() because some implementations return types that are iterable,
# but not iterators such as lists or tuples
slices = iter(stream.stream_slices(sync_mode=sync_mode, cursor_field=stream.cursor_field, stream_state=stream_state))
return next(slices)
def get_first_record_for_slice(
stream: Stream, sync_mode: SyncMode, stream_slice: Optional[Mapping[str, Any]], stream_state: Optional[Mapping[str, Any]]
) -> StreamData:
"""
Gets the first record for a stream_slice of a stream.
:param stream: stream
:param sync_mode: sync_mode
:param stream_slice: stream_slice
:param stream_state: stream_state
:raises StopIteration: if there is no first record to return (the read_records generator is empty)
:return: StreamData containing the first record in the slice
"""
# We wrap the return output of read_records() because some implementations return types that are iterable,
# but not iterators such as lists or tuples
records_for_slice = iter(stream.read_records(sync_mode=sync_mode, stream_slice=stream_slice, stream_state=stream_state))
return next(records_for_slice)

72
airbyte-integrations/connectors/source-stripe/source_stripe/streams.py
View File

@@ -11,13 +11,15 @@ from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optio
import pendulum
import requests
from airbyte_cdk import BackoffStrategy
from airbyte_cdk.models import SyncMode
from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import ExponentialBackoffStrategy
from airbyte_cdk.sources.streams.core import StreamData
from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream
from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
from source_stripe.availability_strategy import StripeAvailabilityStrategy, StripeSubStreamAvailabilityStrategy
from source_stripe.error_handlers import ParentIncrementalStripeSubStreamErrorHandler, StripeErrorHandler
from source_stripe.error_mappings import PARENT_INCREMENTAL_STRIPE_SUB_STREAM_ERROR_MAPPING
STRIPE_API_VERSION = "2022-11-15"
CACHE_DISABLED = os.environ.get("CACHE_DISABLED")
@@ -92,9 +94,8 @@ class StripeStream(HttpStream, ABC):
DEFAULT_SLICE_RANGE = 365
transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
@property
def availability_strategy(self) -> Optional[AvailabilityStrategy]:
return StripeAvailabilityStrategy()
def get_error_handler(self) -> Optional[ErrorHandler]:
return StripeErrorHandler(logger=self.logger)
@property
def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]:
@@ -124,6 +125,9 @@ class StripeStream(HttpStream, ABC):
return self._extra_request_params(self, *args, **kwargs)
return self._extra_request_params or {}
def get_backoff_strategy(self) -> Optional[Union[BackoffStrategy, List[BackoffStrategy]]]:
return ExponentialBackoffStrategy(config={}, parameters={}, factor=1)
@property
def record_extractor(self) -> IRecordExtractor:
return self._record_extractor
@@ -227,10 +231,10 @@ class CreatedCursorIncrementalStripeStream(StripeStream):
cursor_field: str = "created",
**kwargs,
):
self._cursor_field = cursor_field
super().__init__(*args, **kwargs)
self.lookback_window_days = lookback_window_days
self.start_date_max_days_from_now = start_date_max_days_from_now
self._cursor_field = cursor_field
def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]:
"""
@@ -443,8 +447,8 @@ class IncrementalStripeStream(StripeStream):
event_types: Optional[List[str]] = None,
**kwargs,
):
super().__init__(*args, **kwargs)
self._cursor_field = cursor_field
super().__init__(*args, **kwargs)
created_cursor_stream = CreatedCursorIncrementalStripeStream(
*args,
cursor_field=cursor_field,
@@ -514,10 +518,6 @@ class CustomerBalanceTransactions(StripeStream):
def path(self, stream_slice: Mapping[str, Any] = None, **kwargs):
return f"customers/{stream_slice['id']}/balance_transactions"
@property
def availability_strategy(self) -> Optional[AvailabilityStrategy]:
return StripeSubStreamAvailabilityStrategy()
def stream_slices(
self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
) -> Iterable[Optional[Mapping[str, Any]]]:
@@ -558,12 +558,6 @@ class SetupAttempts(CreatedCursorIncrementalStripeStream, HttpSubStream):
def path(self, **kwargs) -> str:
return "setup_attempts"
@property
def availability_strategy(self) -> Optional[AvailabilityStrategy]:
# we use the default http availability strategy here because parent stream may lack data in the incremental stream mode
# and this stream would be marked inaccessible which is not actually true
return HttpAvailabilityStrategy()
def stream_slices(
self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
) -> Iterable[Optional[Mapping[str, Any]]]:
@@ -603,10 +597,6 @@ class Persons(UpdatedCursorIncrementalStripeStream, HttpSubStream):
parent = StripeStream(*args, name="accounts", path="accounts", use_cache=USE_CACHE, **kwargs)
super().__init__(*args, parent=parent, **kwargs)
@property
def availability_strategy(self) -> Optional[AvailabilityStrategy]:
return StripeSubStreamAvailabilityStrategy()
def path(self, stream_slice: Mapping[str, Any] = None, **kwargs):
return f"accounts/{stream_slice['parent']['id']}/persons"
@@ -618,9 +608,7 @@ class Persons(UpdatedCursorIncrementalStripeStream, HttpSubStream):
class StripeSubStream(StripeStream, HttpSubStream):
@property
def availability_strategy(self) -> Optional[AvailabilityStrategy]:
return StripeSubStreamAvailabilityStrategy()
pass
class StripeLazySubStream(StripeStream, HttpSubStream):
@@ -682,10 +670,6 @@ class StripeLazySubStream(StripeStream, HttpSubStream):
super().__init__(*args, **kwargs)
self._sub_items_attr = sub_items_attr
@property
def availability_strategy(self) -> Optional[AvailabilityStrategy]:
return StripeSubStreamAvailabilityStrategy()
def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs):
params = super().request_params(stream_slice=stream_slice, **kwargs)
@@ -734,8 +718,8 @@ class UpdatedCursorIncrementalStripeLazySubStream(StripeStream, ABC):
response_filter: Optional[Callable] = None,
**kwargs,
):
super().__init__(*args, **kwargs)
self._cursor_field = cursor_field
super().__init__(*args, **kwargs)
self.updated_cursor_incremental_stream = UpdatedCursorIncrementalStripeStream(
*args,
cursor_field=cursor_field,
@@ -823,27 +807,7 @@ class ParentIncrementalStipeSubStream(StripeSubStream):
def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]:
return {self.cursor_field: max(current_stream_state.get(self.cursor_field, 0), latest_record[self.cursor_field])}
@property
def raise_on_http_errors(self) -> bool:
return False
def parse_response(self, response: requests.Response, *args, **kwargs) -> Iterable[Mapping[str, Any]]:
if response.status_code == 200:
return super().parse_response(response, *args, **kwargs)
if response.status_code == 404:
# When running incremental sync with state, the returned parent object very likely will not contain sub-items
# as the events API does not support expandable items. Parent class will try getting sub-items from this object,
# then from its own API. In case there are no sub-items at all for this entity, API will raise 404 error.
self.logger.warning(
f"Data was not found for URL: {response.request.url}. "
"If this is a path for getting child attributes like /v1/checkout/sessions/<session_id>/line_items when running "
"the incremental sync, you may safely ignore this warning."
)
return []
response.raise_for_status()
@property
def availability_strategy(self) -> Optional[AvailabilityStrategy]:
# we use the default http availability strategy here because parent stream may lack data in the incremental stream mode
# and this stream would be marked inaccessible which is not actually true
return HttpAvailabilityStrategy()
def get_error_handler(self) -> Optional[ErrorHandler]:
return ParentIncrementalStripeSubStreamErrorHandler(
logger=self.logger, error_mapping=PARENT_INCREMENTAL_STRIPE_SUB_STREAM_ERROR_MAPPING
)

27
airbyte-integrations/connectors/source-stripe/unit_tests/integration/helpers.py Normal file
View File

@@ -0,0 +1,27 @@
#
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
#
from typing import Optional
from airbyte_cdk.models import AirbyteStreamStatus
def assert_stream_did_not_run(output, stream_name: str, expected_error_message_pattern: Optional[str]=None):
# right now, no stream status AirbyteStreamStatus.RUNNING means stream not running
expected = [
AirbyteStreamStatus.STARTED,
AirbyteStreamStatus.COMPLETE,
]
assert output.get_stream_statuses(stream_name) == expected
assert output.records == []
if expected_error_message_pattern:
def contains_substring(message, expected_message_pattern):
return expected_message_pattern in message.log.message
# Use any to check if any message contains the substring
found = any(contains_substring(message, expected_error_message_pattern) for message in output.logs)
assert found, f"Expected message '{expected_error_message_pattern}' not found in logs."

79
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py
View File

@@ -3,12 +3,13 @@
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.sources.source import TState
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
from airbyte_cdk.test.mock_http import HttpMocker
from airbyte_cdk.test.mock_http.response_builder import (
FieldPath,
HttpResponseBuilder,
@@ -19,16 +20,9 @@ from airbyte_cdk.test.mock_http.response_builder import (
find_template,
)
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import (
AirbyteStateBlob,
AirbyteStateMessage,
AirbyteStreamState,
ConfiguredAirbyteCatalog,
FailureType,
StreamDescriptor,
SyncMode,
)
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -101,20 +95,6 @@ def _application_fees_response() -> HttpResponseBuilder:
)
def _given_application_fees_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_application_fees_response().build()
)
def _given_events_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().build()
)
def _read(
config_builder: ConfigBuilder,
sync_mode: SyncMode,
@@ -131,7 +111,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_application_fees_response().with_record(_an_application_fee()).with_record(_an_application_fee()).build(),
@@ -143,7 +122,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_application_fees_response().with_pagination().with_record(_an_application_fee().with_id("last_record_id_from_first_page")).build(),
@@ -159,7 +137,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_application_fees_response().with_record(_an_application_fee()).build(),
@@ -171,7 +148,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_application_fees_response().with_record(_an_application_fee()).build(),
@@ -187,7 +163,6 @@ class FullRefreshTest(TestCase):
slice_range = timedelta(days=20)
slice_datetime = start_date + slice_range
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(),
_application_fees_response().build(),
@@ -202,26 +177,27 @@ class FullRefreshTest(TestCase):
# request matched http_mocker
@HttpMocker()
def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_application_fees_request().with_any_query_params().build(),
a_response_with_status(400),
)
output = self._read(_config())
assert len(output.get_stream_statuses(_STREAM_NAME)) == 0
assert_stream_did_not_run(output, _STREAM_NAME, "Your account is not set up to use Issuing")
@HttpMocker()
def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_application_fees_request().with_any_query_params().build(),
a_response_with_status(401),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
# entrypoint caught unexpected exception as availability strategy was just raising http exceptions for not 400 and 403.
# now default_error_mapping will classify 401 as config error which seems correct
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_any_query_params().build(),
[
@@ -234,7 +210,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_any_query_params().build(),
[a_response_with_status(500), _application_fees_response().with_record(_an_application_fee()).build()],
@@ -243,30 +218,16 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 1
@HttpMocker()
def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_application_fees_request().with_any_query_params().build(),
a_response_with_status(500),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
with patch.object(HttpStatusErrorHandler, 'max_retries', new=0):
output = self._read(_config(), expecting_exception=True)
# concurrent read processor handles exceptions as config errors after complete the max_retries
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None:
# see https://github.com/airbytehq/airbyte/issues/33499
events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build()
http_mocker.get(
events_requests,
_events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now
)
http_mocker.get(
_application_fees_request().with_any_query_params().build(),
_application_fees_response().build(),
)
self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1))
http_mocker.assert_number_of_calls(events_requests, 30)
def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception)
@@ -277,7 +238,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_use_application_fees_endpoint(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
cursor_value = int(_A_START_DATE.timestamp()) + 1
http_mocker.get(
_application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
@@ -294,8 +254,6 @@ class IncrementalTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_value = int(state_datetime.timestamp()) + 1
_given_application_fees_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(
@@ -314,8 +272,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_application_fees_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
state_datetime = _NOW - timedelta(days=5)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
@@ -343,8 +299,6 @@ class IncrementalTest(TestCase):
slice_range = timedelta(days=3)
slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range
_given_application_fees_availability_check(http_mocker)
_given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(self._an_application_fee_event()).build(),
@@ -365,7 +319,6 @@ class IncrementalTest(TestCase):
def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None:
# this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the
# application fees endpoint
_given_application_fees_availability_check(http_mocker)
start_date = _NOW - timedelta(days=40)
state_value = _NOW - timedelta(days=39)
events_lower_boundary = _NOW - timedelta(days=30)

59
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py
View File

@@ -5,12 +5,14 @@ import json
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.sources.source import TState
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
from airbyte_cdk.test.mock_http import HttpMocker
from airbyte_cdk.test.mock_http.response_builder import (
FieldPath,
HttpResponseBuilder,
@@ -21,8 +23,9 @@ from airbyte_cdk.test.mock_http.response_builder import (
find_template,
)
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStreamState, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from airbyte_protocol.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -118,20 +121,6 @@ def _refunds_response() -> HttpResponseBuilder:
)
def _given_application_fees_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_application_fees_response().with_record(_an_application_fee()).build() # there needs to be a record in the parent stream for the child to be available
)
def _given_events_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().build()
)
def _as_dict(response_builder: HttpResponseBuilder) -> Dict[str, Any]:
return json.loads(response_builder.build().body)
@@ -147,16 +136,10 @@ def _read(
return read(_source(catalog, config, state), config, catalog, state, expecting_exception)
def _assert_not_available(output: EntrypointOutput) -> None:
# right now, no stream statuses means stream unavailable
assert len(output.get_stream_statuses(_STREAM_NAME)) == 0
@freezegun.freeze_time(_NOW.isoformat())
class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_application_fees_response()
@@ -183,7 +166,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_multiple_refunds_pages_when_read_then_query_pagination_on_child(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_application_fees_response()
@@ -215,7 +197,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_multiple_application_fees_pages_when_read_then_query_pagination_on_parent(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_application_fees_response()
@@ -252,7 +233,7 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 2
@HttpMocker()
def test_given_parent_stream_without_refund_when_read_then_stream_is_unavailable(self, http_mocker: HttpMocker) -> None:
def test_given_parent_stream_without_refund_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None:
# events stream is not validated as application fees is validated first
http_mocker.get(
_application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
@@ -260,8 +241,7 @@ class FullRefreshTest(TestCase):
)
output = self._read(_config().with_start_date(_A_START_DATE))
_assert_not_available(output)
assert_stream_did_not_run(output, _STREAM_NAME)
@HttpMocker()
def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None:
@@ -269,7 +249,6 @@ class FullRefreshTest(TestCase):
slice_range = timedelta(days=20)
slice_datetime = start_date + slice_range
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(),
_application_fees_response().with_record(
@@ -298,7 +277,6 @@ class FullRefreshTest(TestCase):
slice_range = timedelta(days=20)
slice_datetime = start_date + slice_range
_given_events_availability_check(http_mocker)
http_mocker.get(
StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_application_fees_response().build()
@@ -330,7 +308,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_application_fees_response().with_record(_an_application_fee()).build(),
@@ -342,7 +319,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_cursor_field_is_set(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_application_fees_response()
@@ -363,17 +339,16 @@ class FullRefreshTest(TestCase):
assert output.records[0].record.data["updated"] == output.records[0].record.data["created"]
@HttpMocker()
def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_application_fees_request().with_any_query_params().build(),
a_response_with_status(401),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_application_fees_request().with_any_query_params().build(),
[
@@ -391,14 +366,16 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 1
@HttpMocker()
def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None:
request = _application_fees_request().with_any_query_params().build()
http_mocker.get(
request,
a_response_with_status(500),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
with patch.object(HttpStatusErrorHandler, 'max_retries', new=0):
output = self._read(_config(), expecting_exception=True)
# concurrent read processor handles exceptions as config errors after complete the max_retries
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception)
@@ -409,7 +386,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_use_application_fees_endpoint(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
cursor_value = int(_A_START_DATE.timestamp()) + 1
http_mocker.get(
_application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
@@ -431,8 +407,6 @@ class IncrementalTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_value = int(state_datetime.timestamp()) + 1
_given_application_fees_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(
@@ -451,8 +425,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_application_fees_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
state_datetime = _NOW - timedelta(days=5)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
@@ -478,8 +450,6 @@ class IncrementalTest(TestCase):
slice_range = timedelta(days=3)
slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range
_given_application_fees_availability_check(http_mocker)
_given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(self._a_refund_event()).build(),
@@ -500,7 +470,6 @@ class IncrementalTest(TestCase):
def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None:
# this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the
# application fees endpoint
_given_application_fees_availability_check(http_mocker)
start_date = _NOW - timedelta(days=40)
state_value = _NOW - timedelta(days=39)
events_lower_boundary = _NOW - timedelta(days=30)

65
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py
View File

@@ -3,12 +3,14 @@
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.sources.source import TState
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
from airbyte_cdk.test.mock_http import HttpMocker
from airbyte_cdk.test.mock_http.response_builder import (
FieldPath,
HttpResponseBuilder,
@@ -21,6 +23,7 @@ from airbyte_cdk.test.mock_http.response_builder import (
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStreamState, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -93,20 +96,6 @@ def _authorizations_response() -> HttpResponseBuilder:
)
def _given_authorizations_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.issuing_authorizations_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_authorizations_response().build()
)
def _given_events_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().build()
)
def _read(
config_builder: ConfigBuilder,
sync_mode: SyncMode,
@@ -123,7 +112,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_authorizations_response().with_record(_an_authorization()).with_record(_an_authorization()).build(),
@@ -135,7 +123,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_authorizations_response().with_pagination().with_record(_an_authorization().with_id("last_record_id_from_first_page")).build(),
@@ -151,7 +138,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_authorizations_response().with_record(_an_authorization()).build(),
@@ -163,7 +149,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_authorizations_response().with_record(_an_authorization()).build(),
@@ -179,7 +164,6 @@ class FullRefreshTest(TestCase):
slice_range = timedelta(days=20)
slice_datetime = start_date + slice_range
_given_events_availability_check(http_mocker)
http_mocker.get(
_authorizations_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(),
_authorizations_response().build(),
@@ -194,26 +178,25 @@ class FullRefreshTest(TestCase):
# request matched http_mocker
@HttpMocker()
def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_authorizations_request().with_any_query_params().build(),
a_response_with_status(400),
)
output = self._read(_config())
assert len(output.get_stream_statuses(_STREAM_NAME)) == 0
assert_stream_did_not_run(output, _STREAM_NAME, "Your account is not set up to use Issuing")
@HttpMocker()
def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_authorizations_request().with_any_query_params().build(),
a_response_with_status(401),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_authorizations_request().with_any_query_params().build(),
[
@@ -226,7 +209,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_authorizations_request().with_any_query_params().build(),
[a_response_with_status(500), _authorizations_response().with_record(_an_authorization()).build()],
@@ -235,30 +217,15 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 1
@HttpMocker()
def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_authorizations_request().with_any_query_params().build(),
a_response_with_status(500),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
with patch.object(HttpStatusErrorHandler, 'max_retries', new=1):
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None:
# see https://github.com/airbytehq/airbyte/issues/33499
events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build()
http_mocker.get(
events_requests,
_events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now
)
http_mocker.get(
_authorizations_request().with_any_query_params().build(),
_authorizations_response().build(),
)
self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1))
http_mocker.assert_number_of_calls(events_requests, 30)
def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception)
@@ -269,7 +236,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_use_authorizations_endpoint(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
cursor_value = int(_A_START_DATE.timestamp()) + 1
http_mocker.get(
_authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
@@ -286,8 +252,6 @@ class IncrementalTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_value = int(state_datetime.timestamp()) + 1
_given_authorizations_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(
@@ -306,8 +270,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_authorizations_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
state_datetime = _NOW - timedelta(days=5)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
@@ -333,8 +295,6 @@ class IncrementalTest(TestCase):
slice_range = timedelta(days=3)
slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range
_given_authorizations_availability_check(http_mocker)
_given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(self._an_authorization_event()).build(),
@@ -355,7 +315,6 @@ class IncrementalTest(TestCase):
def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None:
# this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the
# authorizations endpoint
_given_authorizations_availability_check(http_mocker)
start_date = _NOW - timedelta(days=40)
state_value = _NOW - timedelta(days=39)
events_lower_boundary = _NOW - timedelta(days=30)

59
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py
View File

@@ -5,12 +5,14 @@ import json
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.sources.source import TState
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
from airbyte_cdk.test.mock_http import HttpMocker
from airbyte_cdk.test.mock_http.response_builder import (
FieldPath,
HttpResponseBuilder,
@@ -23,6 +25,7 @@ from airbyte_cdk.test.mock_http.response_builder import (
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStreamState, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -121,20 +124,6 @@ def _bank_accounts_response() -> HttpResponseBuilder:
)
def _given_customers_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_customers_response().with_record(_a_customer()).build() # there needs to be a record in the parent stream for the child to be available
)
def _given_events_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().build()
)
def _as_dict(response_builder: HttpResponseBuilder) -> Dict[str, Any]:
return json.loads(response_builder.build().body)
@@ -150,16 +139,10 @@ def _read(
return read(_source(catalog, config, state), config, catalog, state, expecting_exception)
def _assert_not_available(output: EntrypointOutput) -> None:
# right now, no stream statuses means stream unavailable
assert len(output.get_stream_statuses(_STREAM_NAME)) == 0
@freezegun.freeze_time(_NOW.isoformat())
class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_customers_response()
@@ -186,7 +169,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_source_is_not_bank_account_when_read_then_filter_record(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_customers_response()
@@ -208,7 +190,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_multiple_bank_accounts_pages_when_read_then_query_pagination_on_child(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_customers_response()
@@ -240,7 +221,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_multiple_customers_pages_when_read_then_query_pagination_on_parent(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_customers_response()
@@ -277,8 +257,7 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 2
@HttpMocker()
def test_given_parent_stream_without_bank_accounts_when_read_then_stream_is_unavailable(self, http_mocker: HttpMocker) -> None:
# events stream is not validated as application fees is validated first
def test_given_parent_stream_without_bank_accounts_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_customers_response().build(),
@@ -286,7 +265,7 @@ class FullRefreshTest(TestCase):
output = self._read(_config().with_start_date(_A_START_DATE))
_assert_not_available(output)
assert_stream_did_not_run(output, _STREAM_NAME)
@HttpMocker()
def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None:
@@ -294,7 +273,6 @@ class FullRefreshTest(TestCase):
slice_range = timedelta(days=20)
slice_datetime = start_date + slice_range
_given_events_availability_check(http_mocker)
http_mocker.get(
_customers_request().with_expands(_EXPANDS).with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(),
_customers_response().with_record(
@@ -323,7 +301,6 @@ class FullRefreshTest(TestCase):
slice_range = timedelta(days=20)
slice_datetime = start_date + slice_range
_given_events_availability_check(http_mocker)
http_mocker.get(
StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_customers_response().build()
@@ -355,7 +332,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_customers_response().with_record(_a_customer()).build(),
@@ -367,7 +343,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_cursor_field_is_set(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_customers_response()
@@ -388,17 +363,16 @@ class FullRefreshTest(TestCase):
assert output.records[0].record.data["updated"] == int(_NOW.timestamp())
@HttpMocker()
def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_customers_request().with_any_query_params().build(),
a_response_with_status(401),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_customers_request().with_any_query_params().build(),
[
@@ -416,14 +390,15 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 1
@HttpMocker()
def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None:
request = _customers_request().with_any_query_params().build()
http_mocker.get(
request,
a_response_with_status(500),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
with patch.object(HttpStatusErrorHandler, 'max_retries', new=1):
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception)
@@ -436,7 +411,6 @@ class IncrementalTest(TestCase):
def test_given_no_state_and_successful_sync_when_read_then_set_state_to_now(self, http_mocker: HttpMocker) -> None:
# If stripe takes some time to ingest the data, we should recommend to use a lookback window when syncing the bank_accounts stream
# to make sure that we don't lose data between the first and the second sync
_given_events_availability_check(http_mocker)
http_mocker.get(
_customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_customers_response().with_record(
@@ -457,8 +431,6 @@ class IncrementalTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_value = int(state_datetime.timestamp()) + 1
_given_customers_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(
@@ -477,8 +449,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_customers_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
state_datetime = _NOW - timedelta(days=5)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
@@ -504,8 +474,6 @@ class IncrementalTest(TestCase):
slice_range = timedelta(days=3)
slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range
_given_customers_availability_check(http_mocker)
_given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(self._a_bank_account_event()).build(),
@@ -526,7 +494,6 @@ class IncrementalTest(TestCase):
def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None:
# this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the
# customer endpoint
_given_customers_availability_check(http_mocker)
start_date = _NOW - timedelta(days=40)
state_value = _NOW - timedelta(days=39)
events_lower_boundary = _NOW - timedelta(days=30)
@@ -544,8 +511,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_source_is_not_bank_account_when_read_then_filter_record(self, http_mocker: HttpMocker) -> None:
_given_customers_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
state_datetime = _NOW - timedelta(days=5)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),

67
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py
View File

@@ -3,12 +3,14 @@
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.sources.source import TState
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
from airbyte_cdk.test.mock_http import HttpMocker
from airbyte_cdk.test.mock_http.response_builder import (
FieldPath,
HttpResponseBuilder,
@@ -21,6 +23,7 @@ from airbyte_cdk.test.mock_http.response_builder import (
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStreamState, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -93,20 +96,6 @@ def _cards_response() -> HttpResponseBuilder:
)
def _given_cards_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.issuing_cards_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_cards_response().build()
)
def _given_events_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().build()
)
def _read(
config_builder: ConfigBuilder,
sync_mode: SyncMode,
@@ -123,7 +112,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_cards_response().with_record(_a_card()).with_record(_a_card()).build(),
@@ -135,7 +123,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_cards_response().with_pagination().with_record(_a_card().with_id("last_record_id_from_first_page")).build(),
@@ -151,7 +138,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_cards_response().with_record(_a_card()).build(),
@@ -163,7 +149,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_cards_response().with_record(_a_card()).build(),
@@ -179,7 +164,6 @@ class FullRefreshTest(TestCase):
slice_range = timedelta(days=20)
slice_datetime = start_date + slice_range
_given_events_availability_check(http_mocker)
http_mocker.get(
_cards_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(),
_cards_response().build(),
@@ -194,26 +178,25 @@ class FullRefreshTest(TestCase):
# request matched http_mocker
@HttpMocker()
def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_cards_request().with_any_query_params().build(),
a_response_with_status(400),
)
output = self._read(_config())
assert len(output.get_stream_statuses(_STREAM_NAME)) == 0
assert_stream_did_not_run(output, _STREAM_NAME, "Your account is not set up to use Issuing")
@HttpMocker()
def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_cards_request().with_any_query_params().build(),
a_response_with_status(401),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_cards_request().with_any_query_params().build(),
[
@@ -226,7 +209,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_cards_request().with_any_query_params().build(),
[a_response_with_status(500), _cards_response().with_record(_a_card()).build()],
@@ -235,41 +217,23 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 1
@HttpMocker()
def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_cards_request().with_any_query_params().build(),
a_response_with_status(500),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
@HttpMocker()
def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None:
# see https://github.com/airbytehq/airbyte/issues/33499
events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build()
http_mocker.get(
events_requests,
_events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now
)
http_mocker.get(
_cards_request().with_any_query_params().build(),
_cards_response().build(),
)
self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1))
http_mocker.assert_number_of_calls(events_requests, 30)
with patch.object(HttpStatusErrorHandler, 'max_retries', new=1):
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception)
@freezegun.freeze_time(_NOW.isoformat())
class IncrementalTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_use_cards_endpoint(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
cursor_value = int(_A_START_DATE.timestamp()) + 1
http_mocker.get(
_cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
@@ -286,8 +250,6 @@ class IncrementalTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_value = int(state_datetime.timestamp()) + 1
_given_cards_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(
@@ -306,8 +268,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_cards_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
state_datetime = _NOW - timedelta(days=5)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
@@ -333,8 +293,6 @@ class IncrementalTest(TestCase):
slice_range = timedelta(days=3)
slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range
_given_cards_availability_check(http_mocker)
_given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(self._a_card_event()).build(),
@@ -355,7 +313,6 @@ class IncrementalTest(TestCase):
def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None:
# this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the
# cards endpoint
_given_cards_availability_check(http_mocker)
start_date = _NOW - timedelta(days=40)
state_value = _NOW - timedelta(days=39)
events_lower_boundary = _NOW - timedelta(days=30)

65
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py
View File

@@ -3,12 +3,14 @@
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.sources.source import TState
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
from airbyte_cdk.test.mock_http import HttpMocker
from airbyte_cdk.test.mock_http.response_builder import (
FieldPath,
HttpResponseBuilder,
@@ -19,8 +21,9 @@ from airbyte_cdk.test.mock_http.response_builder import (
find_template,
)
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStreamState, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from airbyte_protocol.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -93,20 +96,6 @@ def _early_fraud_warnings_response() -> HttpResponseBuilder:
)
def _given_early_fraud_warnings_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.radar_early_fraud_warnings_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_early_fraud_warnings_response().build()
)
def _given_events_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().build()
)
def _read(
config_builder: ConfigBuilder,
sync_mode: SyncMode,
@@ -123,7 +112,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_early_fraud_warnings_request().with_limit(100).build(),
_early_fraud_warnings_response().with_record(_an_early_fraud_warning()).with_record(_an_early_fraud_warning()).build(),
@@ -135,7 +123,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_early_fraud_warnings_request().with_limit(100).build(),
_early_fraud_warnings_response().with_pagination().with_record(_an_early_fraud_warning().with_id("last_record_id_from_first_page")).build(),
@@ -151,7 +138,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_early_fraud_warnings_request().with_limit(100).build(),
_early_fraud_warnings_response().with_record(_an_early_fraud_warning()).build(),
@@ -162,26 +148,25 @@ class FullRefreshTest(TestCase):
assert output.records[0].record.data["updated"] == output.records[0].record.data["created"]
@HttpMocker()
def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_early_fraud_warnings_request().with_any_query_params().build(),
a_response_with_status(400),
)
output = self._read(_config())
assert len(output.get_stream_statuses(_STREAM_NAME)) == 0
assert_stream_did_not_run(output, _STREAM_NAME, "Your account is not set up to use Issuing")
@HttpMocker()
def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_early_fraud_warnings_request().with_any_query_params().build(),
a_response_with_status(401),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_early_fraud_warnings_request().with_any_query_params().build(),
[
@@ -194,7 +179,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_early_fraud_warnings_request().with_any_query_params().build(),
[a_response_with_status(500), _early_fraud_warnings_response().with_record(_an_early_fraud_warning()).build()],
@@ -203,30 +187,15 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 1
@HttpMocker()
def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_early_fraud_warnings_request().with_any_query_params().build(),
a_response_with_status(500),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
with patch.object(HttpStatusErrorHandler, 'max_retries', new=1):
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None:
# see https://github.com/airbytehq/airbyte/issues/33499
events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build()
http_mocker.get(
events_requests,
_events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now
)
http_mocker.get(
_early_fraud_warnings_request().with_any_query_params().build(),
_early_fraud_warnings_response().build(),
)
self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1))
http_mocker.assert_number_of_calls(events_requests, 30)
def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception)
@@ -237,7 +206,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_use_early_fraud_warnings_endpoint(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
cursor_value = int(_A_START_DATE.timestamp()) + 1
http_mocker.get(
_early_fraud_warnings_request().with_limit(100).build(),
@@ -254,8 +222,6 @@ class IncrementalTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_value = int(state_datetime.timestamp()) + 1
_given_early_fraud_warnings_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(
@@ -274,8 +240,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_early_fraud_warnings_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
state_datetime = _NOW - timedelta(days=5)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
@@ -301,8 +265,6 @@ class IncrementalTest(TestCase):
slice_range = timedelta(days=3)
slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range
_given_early_fraud_warnings_availability_check(http_mocker)
_given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(self._an_early_fraud_warning_event()).build(),
@@ -323,7 +285,6 @@ class IncrementalTest(TestCase):
def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None:
# this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the
# radar/early_fraud_warnings endpoint
_given_early_fraud_warnings_availability_check(http_mocker)
start_date = _NOW - timedelta(days=40)
state_value = _NOW - timedelta(days=39)
events_lower_boundary = _NOW - timedelta(days=30)

40
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py
View File

@@ -3,12 +3,14 @@
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.sources.source import TState
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse
from airbyte_cdk.test.mock_http import HttpMocker
from airbyte_cdk.test.mock_http.response_builder import (
FieldPath,
HttpResponseBuilder,
@@ -18,8 +20,9 @@ from airbyte_cdk.test.mock_http.response_builder import (
find_template,
)
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStreamState, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from airbyte_protocol.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -158,13 +161,13 @@ class FullRefreshTest(TestCase):
self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days))
@HttpMocker()
def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_a_request().with_any_query_params().build(),
a_response_with_status(400),
)
output = self._read(_config())
assert len(output.get_stream_statuses(_STREAM_NAME)) == 0
assert_stream_did_not_run(output, _STREAM_NAME, "Your account is not set up to use Issuing")
@HttpMocker()
def test_given_http_status_401_when_read_then_stream_is_incomplete(self, http_mocker: HttpMocker) -> None:
@@ -173,7 +176,7 @@ class FullRefreshTest(TestCase):
a_response_with_status(401),
)
output = self._read(_config().with_start_date(_A_START_DATE), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
@@ -197,23 +200,24 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 1
@HttpMocker()
def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_a_request().with_any_query_params().build(),
a_response_with_status(500),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
with patch.object(HttpStatusErrorHandler, 'max_retries', new=0):
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_when_read_then_validate_availability_for_full_refresh_and_incremental(self, http_mocker: HttpMocker) -> None:
def test_when_read(self, http_mocker: HttpMocker) -> None:
request = _a_request().with_any_query_params().build()
http_mocker.get(
request,
_a_response().build(),
)
self._read(_config().with_start_date(_A_START_DATE))
http_mocker.assert_number_of_calls(request, 3) # one call for full_refresh availability, one call for incremental availability and one call for the actual read
http_mocker.assert_number_of_calls(request, 1) # one call for the actual read
def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception)
@@ -237,11 +241,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_state_when_read_then_use_state_for_query_params(self, http_mocker: HttpMocker) -> None:
state_value = _A_START_DATE + timedelta(seconds=1)
availability_check_requests = _a_request().with_any_query_params().build()
http_mocker.get(
availability_check_requests,
_a_response().with_record(_a_record()).build(),
)
http_mocker.get(
_a_request().with_created_gte(state_value + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(),
_a_response().with_record(_a_record()).build(),
@@ -260,21 +259,16 @@ class IncrementalTest(TestCase):
We do not see exactly how this case can happen in a real life scenario but it is used to see if at least one state message
would be populated given that no partitions were created.
"""
cursor_value = int(_A_START_DATE.timestamp()) + 1
more_recent_than_record_cursor = int(_NOW.timestamp()) - 1
http_mocker.get(
_a_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_a_response().with_record(_a_record().with_cursor(cursor_value)).build(),
)
very_recent_cursor_state = int(_NOW.timestamp()) - 1
output = self._read(
_config().with_start_date(_A_START_DATE),
StateBuilder().with_stream_state("events", {"created": more_recent_than_record_cursor}).build()
StateBuilder().with_stream_state("events", {"created": very_recent_cursor_state}).build()
)
most_recent_state = output.most_recent_state
assert most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME)
assert most_recent_state.stream_state == AirbyteStateBlob(created=more_recent_than_record_cursor)
assert most_recent_state.stream_state == AirbyteStateBlob(created=very_recent_cursor_state)
def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.incremental, state, expecting_exception)

66
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py
View File

@@ -3,12 +3,14 @@
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.sources.source import TState
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
from airbyte_cdk.test.mock_http import HttpMocker
from airbyte_cdk.test.mock_http.response_builder import (
FieldPath,
HttpResponseBuilder,
@@ -19,8 +21,9 @@ from airbyte_cdk.test.mock_http.response_builder import (
find_template,
)
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStreamState, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from airbyte_protocol.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -93,20 +96,6 @@ def _external_bank_accounts_response() -> HttpResponseBuilder:
)
def _given_external_accounts_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.external_accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_external_bank_accounts_response().build()
)
def _given_events_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().build()
)
def _read(
config_builder: ConfigBuilder,
sync_mode: SyncMode,
@@ -123,7 +112,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_external_accounts_request().with_object(_OBJECT).with_limit(100).build(),
_external_bank_accounts_response().with_record(_an_external_bank_account()).with_record(_an_external_bank_account()).build(),
@@ -135,7 +123,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_external_accounts_request().with_object(_OBJECT).with_limit(100).build(),
_external_bank_accounts_response().with_pagination().with_record(_an_external_bank_account().with_id("last_record_id_from_first_page")).build(),
@@ -151,7 +138,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_external_accounts_request().with_object(_OBJECT).with_limit(100).build(),
_external_bank_accounts_response().with_record(_an_external_bank_account()).build(),
@@ -162,26 +148,25 @@ class FullRefreshTest(TestCase):
assert output.records[0].record.data["updated"] == int(_NOW.timestamp())
@HttpMocker()
def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_external_accounts_request().with_any_query_params().build(),
a_response_with_status(400),
)
output = self._read(_config())
assert len(output.get_stream_statuses(_STREAM_NAME)) == 0
assert_stream_did_not_run(output, _STREAM_NAME, "Your account is not set up to use Issuing")
@HttpMocker()
def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_external_accounts_request().with_any_query_params().build(),
a_response_with_status(401),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_external_accounts_request().with_any_query_params().build(),
[
@@ -194,7 +179,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_external_accounts_request().with_any_query_params().build(),
[a_response_with_status(500), _external_bank_accounts_response().with_record(_an_external_bank_account()).build()],
@@ -203,30 +187,15 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 1
@HttpMocker()
def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_external_accounts_request().with_any_query_params().build(),
a_response_with_status(500),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
with patch.object(HttpStatusErrorHandler, 'max_retries', new =0):
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None:
# see https://github.com/airbytehq/airbyte/issues/33499
events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build()
http_mocker.get(
events_requests,
_events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now
)
http_mocker.get(
_external_accounts_request().with_any_query_params().build(),
_external_bank_accounts_response().build(),
)
self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1))
http_mocker.assert_number_of_calls(events_requests, 30)
def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception)
@@ -237,7 +206,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_use_external_accounts_endpoint(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_external_accounts_request().with_object(_OBJECT).with_limit(100).build(),
_external_bank_accounts_response().with_record(_an_external_bank_account()).build(),
@@ -253,8 +221,6 @@ class IncrementalTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_value = int(state_datetime.timestamp()) + 1
_given_external_accounts_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(
@@ -276,7 +242,6 @@ class IncrementalTest(TestCase):
start_date = _NOW - timedelta(days=40)
state_datetime = _NOW - timedelta(days=5)
_given_external_accounts_availability_check(http_mocker)
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().with_record(
@@ -293,8 +258,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_external_accounts_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
state_datetime = _NOW - timedelta(days=5)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
@@ -320,8 +283,6 @@ class IncrementalTest(TestCase):
slice_range = timedelta(days=3)
slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range
_given_external_accounts_availability_check(http_mocker)
_given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(self._an_external_account_event()).build(),
@@ -342,7 +303,6 @@ class IncrementalTest(TestCase):
def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None:
# this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the
# external_accounts endpoint
_given_external_accounts_availability_check(http_mocker)
start_date = _NOW - timedelta(days=40)
state_value = _NOW - timedelta(days=39)
events_lower_boundary = _NOW - timedelta(days=30)

66
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py
View File

@@ -3,12 +3,14 @@
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.sources.source import TState
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
from airbyte_cdk.test.mock_http import HttpMocker
from airbyte_cdk.test.mock_http.response_builder import (
FieldPath,
HttpResponseBuilder,
@@ -19,8 +21,9 @@ from airbyte_cdk.test.mock_http.response_builder import (
find_template,
)
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStreamState, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from airbyte_protocol.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -98,20 +101,6 @@ def _external_accounts_card_response() -> HttpResponseBuilder:
)
def _given_external_accounts_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.external_accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_external_accounts_card_response().build()
)
def _given_events_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().build()
)
def _read(
config_builder: ConfigBuilder,
sync_mode: SyncMode,
@@ -128,7 +117,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_external_accounts_request().with_object(_OBJECT).with_limit(100).build(),
_external_accounts_card_response().with_record(_an_external_account_card()).with_record(_an_external_account_card()).build(),
@@ -140,7 +128,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_external_accounts_request().with_object(_OBJECT).with_limit(100).build(),
_external_accounts_card_response().with_pagination().with_record(_an_external_account_card().with_id("last_record_id_from_first_page")).build(),
@@ -156,7 +143,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_external_accounts_request().with_object(_OBJECT).with_limit(100).build(),
_external_accounts_card_response().with_record(_an_external_account_card()).build(),
@@ -167,26 +153,25 @@ class FullRefreshTest(TestCase):
assert output.records[0].record.data["updated"] == int(_NOW.timestamp())
@HttpMocker()
def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_external_accounts_request().with_any_query_params().build(),
a_response_with_status(400),
)
output = self._read(_config())
assert len(output.get_stream_statuses(_STREAM_NAME)) == 0
assert_stream_did_not_run(output, _STREAM_NAME, "Your account is not set up to use Issuing")
@HttpMocker()
def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_external_accounts_request().with_any_query_params().build(),
a_response_with_status(401),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_external_accounts_request().with_any_query_params().build(),
[
@@ -199,7 +184,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_external_accounts_request().with_any_query_params().build(),
[a_response_with_status(500), _external_accounts_card_response().with_record(_an_external_account_card()).build()],
@@ -208,30 +192,15 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 1
@HttpMocker()
def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_external_accounts_request().with_any_query_params().build(),
a_response_with_status(500),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
with patch.object(HttpStatusErrorHandler, "max_retries", new=0):
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None:
# see https://github.com/airbytehq/airbyte/issues/33499
events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build()
http_mocker.get(
events_requests,
_events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now
)
http_mocker.get(
_external_accounts_request().with_any_query_params().build(),
_external_accounts_card_response().build(),
)
self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1))
http_mocker.assert_number_of_calls(events_requests, 30)
def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception)
@@ -242,7 +211,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_use_external_accounts_endpoint(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_external_accounts_request().with_object(_OBJECT).with_limit(100).build(),
_external_accounts_card_response().with_record(_an_external_account_card()).build(),
@@ -258,8 +226,6 @@ class IncrementalTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_value = int(state_datetime.timestamp()) + 1
_given_external_accounts_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(
@@ -281,7 +247,6 @@ class IncrementalTest(TestCase):
start_date = _NOW - timedelta(days=40)
state_datetime = _NOW - timedelta(days=5)
_given_external_accounts_availability_check(http_mocker)
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().with_record(
@@ -298,8 +263,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_external_accounts_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
state_datetime = _NOW - timedelta(days=5)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
@@ -325,8 +288,6 @@ class IncrementalTest(TestCase):
slice_range = timedelta(days=3)
slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range
_given_external_accounts_availability_check(http_mocker)
_given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(self._an_external_account_event()).build(),
@@ -347,7 +308,6 @@ class IncrementalTest(TestCase):
def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None:
# this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the
# external_accounts endpoint
_given_external_accounts_availability_check(http_mocker)
start_date = _NOW - timedelta(days=40)
state_value = _NOW - timedelta(days=39)
events_lower_boundary = _NOW - timedelta(days=30)

65
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py
View File

@@ -3,12 +3,14 @@
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.sources.source import TState
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
from airbyte_cdk.test.mock_http import HttpMocker
from airbyte_cdk.test.mock_http.response_builder import (
FieldPath,
HttpResponseBuilder,
@@ -19,8 +21,9 @@ from airbyte_cdk.test.mock_http.response_builder import (
find_template,
)
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStreamState, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from airbyte_protocol.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -98,20 +101,6 @@ def _payment_methods_response() -> HttpResponseBuilder:
)
def _given_payment_methods_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.payment_methods_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_payment_methods_response().build()
)
def _given_events_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().build()
)
def _read(
config_builder: ConfigBuilder,
sync_mode: SyncMode,
@@ -128,7 +117,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_payment_methods_request().with_limit(100).build(),
_payment_methods_response().with_record(_a_payment_method()).with_record(_a_payment_method()).build(),
@@ -140,7 +128,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_payment_methods_request().with_limit(100).build(),
_payment_methods_response().with_pagination().with_record(_a_payment_method().with_id("last_record_id_from_first_page")).build(),
@@ -156,7 +143,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_payment_methods_request().with_limit(100).build(),
_payment_methods_response().with_record(_a_payment_method()).build(),
@@ -167,26 +153,25 @@ class FullRefreshTest(TestCase):
assert output.records[0].record.data["updated"] == output.records[0].record.data["created"]
@HttpMocker()
def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_payment_methods_request().with_any_query_params().build(),
a_response_with_status(400),
)
output = self._read(_config())
assert len(output.get_stream_statuses(_STREAM_NAME)) == 0
assert_stream_did_not_run(output, _STREAM_NAME, "Your account is not set up to use Issuing")
@HttpMocker()
def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_payment_methods_request().with_any_query_params().build(),
a_response_with_status(401),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_payment_methods_request().with_any_query_params().build(),
[
@@ -199,7 +184,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_payment_methods_request().with_any_query_params().build(),
[a_response_with_status(500), _payment_methods_response().with_record(_a_payment_method()).build()],
@@ -208,30 +192,15 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 1
@HttpMocker()
def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_payment_methods_request().with_any_query_params().build(),
a_response_with_status(500),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
with patch.object(HttpStatusErrorHandler, 'max_retries', new=0):
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None:
# see https://github.com/airbytehq/airbyte/issues/33499
events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build()
http_mocker.get(
events_requests,
_events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now
)
http_mocker.get(
_payment_methods_request().with_any_query_params().build(),
_payment_methods_response().build(),
)
self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1))
http_mocker.assert_number_of_calls(events_requests, 30)
def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception)
@@ -242,7 +211,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_use_payment_methods_endpoint(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
cursor_value = int(_A_START_DATE.timestamp()) + 1
http_mocker.get(
_payment_methods_request().with_limit(100).build(),
@@ -259,8 +227,6 @@ class IncrementalTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_value = int(state_datetime.timestamp()) + 1
_given_payment_methods_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(
@@ -279,8 +245,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_payment_methods_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
state_datetime = _NOW - timedelta(days=5)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
@@ -306,8 +270,6 @@ class IncrementalTest(TestCase):
slice_range = timedelta(days=3)
slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range
_given_payment_methods_availability_check(http_mocker)
_given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(self._a_payment_method_event()).build(),
@@ -328,7 +290,6 @@ class IncrementalTest(TestCase):
def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None:
# this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the
# payment_methods endpoint
_given_payment_methods_availability_check(http_mocker)
start_date = _NOW - timedelta(days=40)
state_value = _NOW - timedelta(days=39)
events_lower_boundary = _NOW - timedelta(days=30)

216
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py
View File

@@ -3,9 +3,10 @@
from datetime import datetime, timedelta, timezone
from typing import List
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.models import FailureType, SyncMode
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import read
from airbyte_cdk.test.mock_http import HttpMocker
@@ -19,17 +20,9 @@ from airbyte_cdk.test.mock_http.response_builder import (
find_template,
)
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import (
AirbyteStateBlob,
AirbyteStreamState,
AirbyteStreamStatus,
ConfiguredAirbyteCatalog,
FailureType,
Level,
StreamDescriptor,
SyncMode,
)
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStreamStatus, FailureType, Level, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -121,11 +114,6 @@ class PersonsTest(TestCase):
_create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(),
)
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(),
)
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
@@ -152,13 +140,6 @@ class PersonsTest(TestCase):
_create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(),
)
# The persons stream makes a final call to events endpoint
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(),
)
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
@@ -186,13 +167,6 @@ class PersonsTest(TestCase):
record=_create_record("persons")).build(),
)
# The persons stream makes a final call to events endpoint
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(),
)
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
@@ -208,12 +182,8 @@ class PersonsTest(TestCase):
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
error_log_messages = [message for message in actual_messages.logs if message.log.level == Level.ERROR]
# For Stripe, streams that get back a 400 or 403 response code are skipped over silently without throwing an error as part of
# this connector's availability strategy
assert len(actual_messages.get_stream_statuses(_STREAM_NAME)) == 0
assert len(error_log_messages) > 0
# For Stripe, streams that get back a 400 or 403 response code are skipped over silently
assert_stream_did_not_run(actual_messages, _STREAM_NAME, "Your account is not set up to use Issuing")
@HttpMocker()
def test_persons_400_error(self, http_mocker: HttpMocker):
@@ -230,12 +200,9 @@ class PersonsTest(TestCase):
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
error_log_messages = [message for message in actual_messages.logs if message.log.level == Level.ERROR]
# For Stripe, streams that get back a 400 or 403 response code are skipped over silently without throwing an error as part of
# this connector's availability strategy. They are however reported in the log messages
assert len(actual_messages.get_stream_statuses(_STREAM_NAME)) == 0
assert len(error_log_messages) > 0
# For Stripe, streams that get back a 400 or 403 response code are skipped over silently
assert_stream_did_not_run(actual_messages, _STREAM_NAME, "Your account is not set up to use Issuing")
@HttpMocker()
def test_accounts_401_error(self, http_mocker: HttpMocker):
@@ -247,7 +214,7 @@ class PersonsTest(TestCase):
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog(), expecting_exception=True)
assert actual_messages.errors[-1].trace.error.failure_type == FailureType.system_error
assert actual_messages.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_persons_401_error(self, http_mocker: HttpMocker):
@@ -265,7 +232,7 @@ class PersonsTest(TestCase):
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog(), expecting_exception=True)
assert actual_messages.errors[-1].trace.error.failure_type == FailureType.system_error
assert actual_messages.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_persons_403_error(self, http_mocker: HttpMocker):
@@ -282,33 +249,15 @@ class PersonsTest(TestCase):
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog(), expecting_exception=True)
error_log_messages = [message for message in actual_messages.logs if message.log.level == Level.ERROR]
# For Stripe, streams that get back a 400 or 403 response code are skipped over silently without throwing an error as part of
# this connector's availability strategy
assert len(actual_messages.get_stream_statuses(_STREAM_NAME)) == 0
assert len(error_log_messages) > 0
# For Stripe, streams that get back a 400 or 403 response code are skipped over silently
assert_stream_did_not_run(actual_messages, _STREAM_NAME, "This application does not have the required permissions")
@HttpMocker()
def test_incremental_with_recent_state(self, http_mocker: HttpMocker):
state_datetime = _NOW - timedelta(days=5)
cursor_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES
http_mocker.get(
_create_accounts_request().with_limit(100).build(),
_create_response().with_record(record=_create_record("accounts")).build(),
)
http_mocker.get(
_create_persons_request().with_limit(100).build(),
_create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(),
)
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_persons_event_record(event_type="person.created")).with_record(record=_create_persons_event_record(event_type="person.created")).build(),
)
http_mocker.get(
_create_events_request().with_created_gte(cursor_datetime).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
@@ -335,21 +284,6 @@ class PersonsTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES
http_mocker.get(
_create_accounts_request().with_limit(100).build(),
_create_response().with_record(record=_create_record("accounts")).build(),
)
http_mocker.get(
_create_persons_request().with_limit(100).build(),
_create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(),
)
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_persons_event_record(event_type="person.created")).with_record(record=_create_persons_event_record(event_type="person.deleted")).build(),
)
http_mocker.get(
_create_events_request().with_created_gte(cursor_datetime).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
@@ -378,16 +312,6 @@ class PersonsTest(TestCase):
state_datetime = _NOW - timedelta(days=15)
config = _create_config().with_start_date(start_datetime).build()
http_mocker.get(
_create_accounts_request().with_limit(100).build(),
_create_response().with_record(record=_create_record("accounts")).build(),
)
http_mocker.get(
_create_persons_request().with_limit(100).build(),
_create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(),
)
http_mocker.get(
_create_events_request().with_created_gte(start_datetime).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
@@ -424,12 +348,6 @@ class PersonsTest(TestCase):
_create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(),
)
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(),
)
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
@@ -451,12 +369,6 @@ class PersonsTest(TestCase):
]
)
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(),
)
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
@@ -468,24 +380,6 @@ class PersonsTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES
http_mocker.get(
_create_accounts_request().with_limit(100).build(),
_create_response().with_record(record=_create_record("accounts")).build(),
)
http_mocker.get(
_create_persons_request().with_limit(100).build(),
_create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(),
)
# Mock when check_availability is run on the persons incremental stream
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_persons_event_record(event_type="person.created")).with_record(
record=_create_persons_event_record(event_type="person.created")).build(),
)
http_mocker.get(
_create_events_request().with_created_gte(cursor_datetime).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
@@ -519,48 +413,22 @@ class PersonsTest(TestCase):
http_mocker.get(
_create_persons_request().with_limit(100).build(),
[
# Used to pass the initial check_availability before starting the sync
_create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(),
a_response_with_status(429), # Returns 429 on all subsequent requests to test the maximum number of retries
]
)
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(),
)
with patch.object(HttpStatusErrorHandler, "max_retries", new=0):
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
# first error is the actual error, second is to break the Python app with code != 0
assert list(map(lambda message: message.trace.error.failure_type, actual_messages.errors)) == [FailureType.system_error, FailureType.config_error]
# first error is the actual error, second is to break the Python app with code != 0
assert list(map(lambda message: message.trace.error.failure_type, actual_messages.errors)) == [FailureType.system_error, FailureType.config_error]
assert "Request rate limit exceeded" in actual_messages.errors[0].trace.error.internal_message
@HttpMocker()
def test_incremental_rate_limit_max_attempts_exceeded(self, http_mocker: HttpMocker) -> None:
state_datetime = _NOW - timedelta(days=5)
cursor_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES
http_mocker.get(
_create_accounts_request().with_limit(100).build(),
_create_response().with_record(record=_create_record("accounts")).build(),
)
http_mocker.get(
_create_persons_request().with_limit(100).build(),
_create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(),
)
# Mock when check_availability is run on the persons incremental stream
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_persons_event_record(event_type="person.created")).with_record(
record=_create_persons_event_record(event_type="person.created")).build(),
)
http_mocker.get(
_create_events_request().with_created_gte(cursor_datetime).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
@@ -569,14 +437,16 @@ class PersonsTest(TestCase):
state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build()
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(sync_mode=SyncMode.incremental), state=state)
actual_messages = read(
source,
config=_CONFIG,
catalog=_create_catalog(sync_mode=SyncMode.incremental),
state=state,
)
with patch.object(HttpStatusErrorHandler, "max_retries", new=0):
actual_messages = read(
source,
config=_CONFIG,
catalog=_create_catalog(sync_mode=SyncMode.incremental),
state=state,
)
assert len(actual_messages.errors) == 2
assert len(actual_messages.errors) == 2
assert "Request rate limit exceeded" in actual_messages.errors[0].trace.error.message
@HttpMocker()
def test_server_error_parent_stream_accounts(self, http_mocker: HttpMocker) -> None:
@@ -593,11 +463,6 @@ class PersonsTest(TestCase):
_create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(),
)
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(),
)
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
@@ -620,11 +485,6 @@ class PersonsTest(TestCase):
]
)
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(),
)
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
@@ -636,26 +496,12 @@ class PersonsTest(TestCase):
def test_server_error_max_attempts_exceeded(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_create_accounts_request().with_limit(100).build(),
_create_response().with_record(record=_create_record("accounts")).build(),
a_response_with_status(500)
)
http_mocker.get(
_create_persons_request().with_limit(100).build(),
[
# Used to pass the initial check_availability before starting the sync
_create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(),
a_response_with_status(500), # Returns 429 on all subsequent requests to test the maximum number of retries
]
)
http_mocker.get(
_create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(
["person.created", "person.updated", "person.deleted"]).build(),
_create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(),
)
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
with patch.object(HttpStatusErrorHandler, "max_retries", new=0):
source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE)
actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
# first error is the actual error, second is to break the Python app with code != 0
assert list(map(lambda message: message.trace.error.failure_type, actual_messages.errors)) == [FailureType.system_error, FailureType.config_error]

67
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py
View File

@@ -3,12 +3,14 @@
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.sources.source import TState
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
from airbyte_cdk.test.mock_http import HttpMocker
from airbyte_cdk.test.mock_http.response_builder import (
FieldPath,
HttpResponseBuilder,
@@ -19,8 +21,9 @@ from airbyte_cdk.test.mock_http.response_builder import (
find_template,
)
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStreamState, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from airbyte_protocol.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -93,20 +96,6 @@ def _reviews_response() -> HttpResponseBuilder:
)
def _given_reviews_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.reviews_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_reviews_response().build()
)
def _given_events_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().build()
)
def _read(
config_builder: ConfigBuilder,
sync_mode: SyncMode,
@@ -123,7 +112,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_reviews_response().with_record(_a_review()).with_record(_a_review()).build(),
@@ -135,7 +123,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_reviews_response().with_pagination().with_record(_a_review().with_id("last_record_id_from_first_page")).build(),
@@ -151,7 +138,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_reviews_response().with_record(_a_review()).build(),
@@ -163,7 +149,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_reviews_response().with_record(_a_review()).build(),
@@ -179,7 +164,6 @@ class FullRefreshTest(TestCase):
slice_range = timedelta(days=20)
slice_datetime = start_date + slice_range
_given_events_availability_check(http_mocker)
http_mocker.get(
_reviews_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(),
_reviews_response().build(),
@@ -194,26 +178,25 @@ class FullRefreshTest(TestCase):
# request matched http_mocker
@HttpMocker()
def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_reviews_request().with_any_query_params().build(),
a_response_with_status(400),
)
output = self._read(_config())
assert len(output.get_stream_statuses(_STREAM_NAME)) == 0
assert_stream_did_not_run(output, _STREAM_NAME)
@HttpMocker()
def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_reviews_request().with_any_query_params().build(),
a_response_with_status(401),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_reviews_request().with_any_query_params().build(),
[
@@ -226,7 +209,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_reviews_request().with_any_query_params().build(),
[a_response_with_status(500), _reviews_response().with_record(_a_review()).build()],
@@ -235,30 +217,15 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 1
@HttpMocker()
def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_reviews_request().with_any_query_params().build(),
a_response_with_status(500),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
with patch.object(HttpStatusErrorHandler, "max_retries", new=0):
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None:
# see https://github.com/airbytehq/airbyte/issues/33499
events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build()
http_mocker.get(
events_requests,
_events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now
)
http_mocker.get(
_reviews_request().with_any_query_params().build(),
_reviews_response().build(),
)
self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1))
http_mocker.assert_number_of_calls(events_requests, 30)
def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception)
@@ -269,7 +236,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_use_reviews_endpoint(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
cursor_value = int(_A_START_DATE.timestamp()) + 1
http_mocker.get(
_reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
@@ -286,8 +252,6 @@ class IncrementalTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_value = int(state_datetime.timestamp()) + 1
_given_reviews_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(
@@ -306,8 +270,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_reviews_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
state_datetime = _NOW - timedelta(days=5)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
@@ -333,8 +295,6 @@ class IncrementalTest(TestCase):
slice_range = timedelta(days=3)
slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range
_given_reviews_availability_check(http_mocker)
_given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(self._a_review_event()).build(),
@@ -355,7 +315,6 @@ class IncrementalTest(TestCase):
def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None:
# this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the
# reviews endpoint
_given_reviews_availability_check(http_mocker)
start_date = _NOW - timedelta(days=40)
state_value = _NOW - timedelta(days=39)
events_lower_boundary = _NOW - timedelta(days=30)

67
airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py
View File

@@ -3,12 +3,14 @@
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from unittest import TestCase
from unittest.mock import patch
import freezegun
from airbyte_cdk.sources.source import TState
from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
from airbyte_cdk.test.mock_http import HttpMocker
from airbyte_cdk.test.mock_http.response_builder import (
FieldPath,
HttpResponseBuilder,
@@ -19,8 +21,9 @@ from airbyte_cdk.test.mock_http.response_builder import (
find_template,
)
from airbyte_cdk.test.state_builder import StateBuilder
from airbyte_protocol.models import AirbyteStateBlob, AirbyteStreamState, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from airbyte_protocol.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode
from integration.config import ConfigBuilder
from integration.helpers import assert_stream_did_not_run
from integration.pagination import StripePaginationStrategy
from integration.request_builder import StripeRequestBuilder
from integration.response_builder import a_response_with_status
@@ -93,20 +96,6 @@ def _transactions_response() -> HttpResponseBuilder:
)
def _given_transactions_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.issuing_transactions_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_transactions_response().build()
)
def _given_events_availability_check(http_mocker: HttpMocker) -> None:
http_mocker.get(
StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(),
_events_response().build()
)
def _read(
config_builder: ConfigBuilder,
sync_mode: SyncMode,
@@ -123,7 +112,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_transactions_response().with_record(_a_transaction()).with_record(_a_transaction()).build(),
@@ -135,7 +123,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_transactions_response().with_pagination().with_record(_a_transaction().with_id("last_record_id_from_first_page")).build(),
@@ -151,7 +138,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_transactions_response().with_record(_a_transaction()).build(),
@@ -163,7 +149,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
_transactions_response().with_record(_a_transaction()).build(),
@@ -179,7 +164,6 @@ class FullRefreshTest(TestCase):
slice_range = timedelta(days=20)
slice_datetime = start_date + slice_range
_given_events_availability_check(http_mocker)
http_mocker.get(
_transactions_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(),
_transactions_response().build(),
@@ -194,26 +178,25 @@ class FullRefreshTest(TestCase):
# request matched http_mocker
@HttpMocker()
def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_transactions_request().with_any_query_params().build(),
a_response_with_status(400),
)
output = self._read(_config())
assert len(output.get_stream_statuses(_STREAM_NAME)) == 0
assert_stream_did_not_run(output, _STREAM_NAME)
@HttpMocker()
def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_transactions_request().with_any_query_params().build(),
a_response_with_status(401),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_transactions_request().with_any_query_params().build(),
[
@@ -226,7 +209,6 @@ class FullRefreshTest(TestCase):
@HttpMocker()
def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
http_mocker.get(
_transactions_request().with_any_query_params().build(),
[a_response_with_status(500), _transactions_response().with_record(_a_transaction()).build()],
@@ -235,30 +217,15 @@ class FullRefreshTest(TestCase):
assert len(output.records) == 1
@HttpMocker()
def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None:
def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None:
http_mocker.get(
_transactions_request().with_any_query_params().build(),
a_response_with_status(500),
)
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.system_error
with patch.object(HttpStatusErrorHandler, "max_retries", new=0):
output = self._read(_config(), expecting_exception=True)
assert output.errors[-1].trace.error.failure_type == FailureType.config_error
@HttpMocker()
def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None:
# see https://github.com/airbytehq/airbyte/issues/33499
events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build()
http_mocker.get(
events_requests,
_events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now
)
http_mocker.get(
_transactions_request().with_any_query_params().build(),
_transactions_response().build(),
)
self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1))
http_mocker.assert_number_of_calls(events_requests, 30)
def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput:
return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception)
@@ -269,7 +236,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_no_state_when_read_then_use_transactions_endpoint(self, http_mocker: HttpMocker) -> None:
_given_events_availability_check(http_mocker)
cursor_value = int(_A_START_DATE.timestamp()) + 1
http_mocker.get(
_transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(),
@@ -286,8 +252,6 @@ class IncrementalTest(TestCase):
state_datetime = _NOW - timedelta(days=5)
cursor_value = int(state_datetime.timestamp()) + 1
_given_transactions_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(
@@ -306,8 +270,6 @@ class IncrementalTest(TestCase):
@HttpMocker()
def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None:
_given_transactions_availability_check(http_mocker)
_given_events_availability_check(http_mocker)
state_datetime = _NOW - timedelta(days=5)
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(),
@@ -333,8 +295,6 @@ class IncrementalTest(TestCase):
slice_range = timedelta(days=3)
slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range
_given_transactions_availability_check(http_mocker)
_given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check
http_mocker.get(
_events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(),
_events_response().with_record(self._a_transaction_event()).build(),
@@ -355,7 +315,6 @@ class IncrementalTest(TestCase):
def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None:
# this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the
# transactions endpoint
_given_transactions_availability_check(http_mocker)
start_date = _NOW - timedelta(days=40)
state_value = _NOW - timedelta(days=39)
events_lower_boundary = _NOW - timedelta(days=30)

225
airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py
View File

@@ -1,225 +0,0 @@
#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#
import logging
import urllib.parse
import pytest
from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
from source_stripe.availability_strategy import STRIPE_ERROR_CODES, StripeSubStreamAvailabilityStrategy
from source_stripe.streams import IncrementalStripeStream, StripeLazySubStream
@pytest.fixture()
def stream_mock(mocker):
def _mocker():
return mocker.Mock(stream_slices=mocker.Mock(return_value=[{}]), read_records=mocker.Mock(return_value=[{}]))
return _mocker
def test_traverse_over_substreams(stream_mock, mocker):
# Mock base HttpAvailabilityStrategy to capture all the check_availability method calls
check_availability_mock = mocker.MagicMock(return_value=(True, None))
cdk_check_availability_mock = mocker.MagicMock(return_value=(True, None))
mocker.patch(
"source_stripe.availability_strategy.StripeAvailabilityStrategy.check_availability", check_availability_mock
)
mocker.patch(
"airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", cdk_check_availability_mock
)
# Prepare tree of nested objects
root = stream_mock()
root.availability_strategy = HttpAvailabilityStrategy()
root.parent = None
child_1 = stream_mock()
child_1.availability_strategy = StripeSubStreamAvailabilityStrategy()
child_1.parent = root
child_1_1 = stream_mock()
child_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy()
child_1_1.parent = child_1
child_1_1_1 = stream_mock()
child_1_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy()
child_1_1_1.parent = child_1_1
# Start traverse
is_available, reason = child_1_1_1.availability_strategy.check_availability(child_1_1_1, mocker.Mock(), mocker.Mock())
assert is_available and reason is None
# Check availability strategy was called once for every nested object
assert check_availability_mock.call_count == 3
assert cdk_check_availability_mock.call_count == 1
# Check each availability strategy was called with proper instance argument
assert id(cdk_check_availability_mock.call_args_list[0].args[0]) == id(root)
assert id(check_availability_mock.call_args_list[0].args[0]) == id(child_1)
assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1_1)
assert id(check_availability_mock.call_args_list[2].args[0]) == id(child_1_1_1)
def test_traverse_over_substreams_failure(stream_mock, mocker):
# Mock base HttpAvailabilityStrategy to capture all the check_availability method calls
check_availability_mock = mocker.MagicMock(side_effect=[(True, None), (False, "child_1")])
mocker.patch(
"source_stripe.availability_strategy.StripeAvailabilityStrategy.check_availability", check_availability_mock
)
# Prepare tree of nested objects
root = stream_mock()
root.availability_strategy = HttpAvailabilityStrategy()
root.parent = None
child_1 = stream_mock()
child_1.availability_strategy = StripeSubStreamAvailabilityStrategy()
child_1.parent = root
child_1_1 = stream_mock()
child_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy()
child_1_1.parent = child_1
child_1_1_1 = stream_mock()
child_1_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy()
child_1_1_1.parent = child_1_1
# Start traverse
is_available, reason = child_1_1_1.availability_strategy.check_availability(child_1_1_1, mocker.Mock(), mocker.Mock())
assert not is_available and reason == "child_1"
# Check availability strategy was called once for every nested object
assert check_availability_mock.call_count == 2
# Check each availability strategy was called with proper instance argument
assert id(check_availability_mock.call_args_list[0].args[0]) == id(child_1)
assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1_1)
def test_substream_availability(mocker, stream_by_name):
check_availability_mock = mocker.MagicMock()
check_availability_mock.return_value = (True, None)
mocker.patch(
"source_stripe.availability_strategy.StripeAvailabilityStrategy.check_availability", check_availability_mock
)
stream = stream_by_name("invoice_line_items")
is_available, reason = stream.availability_strategy.check_availability(stream, mocker.Mock(), mocker.Mock())
assert is_available and reason is None
assert check_availability_mock.call_count == 2
assert isinstance(check_availability_mock.call_args_list[0].args[0], IncrementalStripeStream)
assert isinstance(check_availability_mock.call_args_list[1].args[0], StripeLazySubStream)
def test_substream_availability_no_parent(mocker, stream_by_name):
check_availability_mock = mocker.MagicMock()
check_availability_mock.return_value = (True, None)
mocker.patch(
"source_stripe.availability_strategy.StripeAvailabilityStrategy.check_availability", check_availability_mock
)
stream = stream_by_name("invoice_line_items")
stream.parent = None
stream.availability_strategy.check_availability(stream, mocker.Mock(), mocker.Mock())
assert check_availability_mock.call_count == 1
assert isinstance(check_availability_mock.call_args_list[0].args[0], StripeLazySubStream)
def test_403_error_handling(stream_by_name, requests_mock):
stream = stream_by_name("invoices")
logger = logging.getLogger("airbyte")
for error_code in STRIPE_ERROR_CODES:
requests_mock.get(f"{stream.url_base}{stream.path()}", status_code=403, json={"error": {"code": f"{error_code}"}})
available, message = stream.check_availability(logger)
assert not available
assert STRIPE_ERROR_CODES[error_code] in message
@pytest.mark.parametrize(
"stream_name, endpoints, expected_calls",
(
(
"accounts",
{
"/v1/accounts": {"data": []}
},
1
),
(
"refunds",
{
"/v1/refunds": {"data": []}, "/v1/events": {"data": []}
},
2
),
(
"credit_notes",
{
"/v1/credit_notes": {"data": []}, "/v1/events": {"data": []}
},
2
),
(
"charges",
{
"/v1/charges": {"data": []}, "/v1/events": {"data": []}
},
2
),
(
"subscription_items",
{
"/v1/subscriptions": {"data": [{"id": 1}]},
"/v1/events": {"data": []}
},
3
),
(
"bank_accounts",
{
"/v1/customers": {"data": [{"id": 1}]},
"/v1/events": {"data": []}
},
2
),
(
"customer_balance_transactions",
{
"/v1/events": {"data": [{"data":{"object": {"id": 1}}, "created": 1, "type": "customer.updated"}]},
"/v1/customers": {"data": [{"id": 1}]},
"/v1/customers/1/balance_transactions": {"data": []}
},
4
),
(
"transfer_reversals",
{
"/v1/transfers": {"data": [{"id": 1}]},
"/v1/events": {"data": [{"data":{"object": {"id": 1}}, "created": 1, "type": "transfer.updated"}]},
"/v1/transfers/1/reversals": {"data": []}
},
4
),
(
"persons",
{
"/v1/accounts": {"data": [{"id": 1}]},
"/v1/events": {"data": []},
"/v1/accounts/1/persons": {"data": []}
},
4
)
)
)
def test_availability_strategy_visits_endpoints(stream_by_name, stream_name, endpoints, expected_calls, requests_mock, mocker, config):
for endpoint, data in endpoints.items():
requests_mock.get(endpoint, json=data)
stream = stream_by_name(stream_name, config)
is_available, reason = stream.check_availability(mocker.Mock(), mocker.Mock())
assert (is_available, reason) == (True, None)
assert len(requests_mock.request_history) == expected_calls
for call in requests_mock.request_history:
assert urllib.parse.urlparse(call.url).path in endpoints.keys()

2
airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py
View File

@@ -152,6 +152,6 @@ def test_call_budget_passed_to_every_stream(mocker):
for stream in streams:
if isinstance(stream, StreamFacade):
stream = stream._legacy_stream
session = stream.request_session()
session = stream._http_client._session
assert isinstance(session, (CachedLimiterSession, LimiterSession))
assert session._api_budget == get_api_call_budget_mock.return_value

1
docs/integrations/sources/stripe.md
View File

@@ -225,6 +225,7 @@ Each record is marked with `is_deleted` flag when the appropriate event happens
| Version | Date | Pull Request | Subject |
| :------ | :--------- | :-------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| 5.5.0 | 2024-08-05 | [43302](https://github.com/airbytehq/airbyte/pull/43302) | Fix problem with state not updating and upgrade cdk 4
| 5.4.12 | 2024-07-31 | [41985](https://github.com/airbytehq/airbyte/pull/41985) | Expand Invoice discounts and tax rates
| 5.4.11 | 2024-07-27 | [42623](https://github.com/airbytehq/airbyte/pull/42623) | Update dependencies |
| 5.4.10 | 2024-07-20 | [42305](https://github.com/airbytehq/airbyte/pull/42305) | Update dependencies |