7ae4f8686b
* Connector health: source hubspot, gitlab, snapchat-marketing: fix builds
* Add stream ShippingRates
* #2363 source mixpanel: reimplement backoff strategy
* #2363 source mixpanel: upd changelog
* #2363 source mixpanel: revert v0.1.32
* rework rate limits
* wait only when running CAT + increase timeouts
* #2363 return backoff time
* code format
* use env variable instead of a hidden field in the config
* review comments
* review fixes
* fix reading env var
* Resolve conflicts, Bump version
* Attempt to bump version to avoid caching
* split test config into to projects
* rollback version
* undo non related changes
* remove non related changes
* revert new line
* fix funnel slice patching
* fix funnel slice patching
* do not use stream state for generating request params!
* fix updating state
* Revert "do not use stream state for generating request params!"
This reverts commit 443781eaa9.
* revert prev commits
* fix timestamp filtering
---------
Co-authored-by: btkcodedev <btk.codedev@gmail.com>
Co-authored-by: Augustin <augustin@airbyte.io>
Co-authored-by: maxi297 <maxime@airbyte.io>
50 lines
1.6 KiB
Python
50 lines
1.6 KiB
Python
#
|
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
#
|
|
|
|
import logging
|
|
import os
|
|
from functools import wraps
|
|
|
|
from .streams import Funnels
|
|
|
|
|
|
def funnel_slices_patched(self: Funnels, sync_mode):
|
|
"""
|
|
Return only first result from funnels
|
|
"""
|
|
funnel_slices_values = self.get_funnel_slices(sync_mode)
|
|
single_slice = next(funnel_slices_values, None)
|
|
return [single_slice] if single_slice else []
|
|
|
|
|
|
def adapt_streams_if_testing(func):
|
|
# Patch Funnels, so we download data only for one Funnel entity
|
|
@wraps(func)
|
|
def wrapper(self, config):
|
|
if bool(os.environ.get("PATCH_FUNNEL_SLICES", "")):
|
|
Funnels.funnel_slices = funnel_slices_patched
|
|
return func(self, config)
|
|
|
|
return wrapper
|
|
|
|
|
|
def adapt_validate_if_testing(func):
|
|
"""
|
|
Due to API limitations (60 requests per hour) it is impossible to run acceptance tests in normal mode,
|
|
so we're reducing amount of requests by aligning start date if `AVAILABLE_TESTING_RANGE_DAYS` flag is set in env variables.
|
|
"""
|
|
|
|
@wraps(func)
|
|
def wrapper(self, config):
|
|
config = func(self, config)
|
|
available_testing_range_days = int(os.environ.get("AVAILABLE_TESTING_RANGE_DAYS", 0))
|
|
if available_testing_range_days:
|
|
logger = logging.getLogger("airbyte")
|
|
logger.info("SOURCE IN TESTING MODE, DO NOT USE IN PRODUCTION!")
|
|
if (config["end_date"] - config["start_date"]).days > available_testing_range_days:
|
|
config["start_date"] = config["end_date"].subtract(days=available_testing_range_days)
|
|
return config
|
|
|
|
return wrapper
|