test(source-klaviyo): Add mock server tests for all streams (do not merge) (#70824)
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: sophie.cui@airbyte.io <sophie.cui@airbyte.io>
This commit is contained in:
committed by
GitHub
parent
21c1ccbf8a
commit
f82cb087cb
@@ -1,3 +1,85 @@
|
||||
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from pytest import fixture
|
||||
|
||||
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
|
||||
from airbyte_cdk.test.catalog_builder import CatalogBuilder
|
||||
from airbyte_cdk.test.state_builder import StateBuilder
|
||||
|
||||
|
||||
pytest_plugins = ["airbyte_cdk.test.utils.manifest_only_fixtures"]
|
||||
|
||||
os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH"
|
||||
|
||||
|
||||
def _get_manifest_path() -> Path:
|
||||
"""
|
||||
Find manifest.yaml location.
|
||||
|
||||
In CI (Docker): /airbyte/integration_code/source_declarative_manifest/manifest.yaml
|
||||
Locally: ../manifest.yaml (relative to unit_tests/)
|
||||
"""
|
||||
ci_path = Path("/airbyte/integration_code/source_declarative_manifest")
|
||||
if ci_path.exists():
|
||||
return ci_path
|
||||
# Use .resolve() to ensure we get an absolute path, as __file__ may be relative in CI
|
||||
return Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
_SOURCE_FOLDER_PATH = _get_manifest_path()
|
||||
_YAML_FILE_PATH = _SOURCE_FOLDER_PATH / "manifest.yaml"
|
||||
|
||||
sys.path.append(str(_SOURCE_FOLDER_PATH))
|
||||
|
||||
|
||||
def get_resource_path(resource_file: str) -> Path:
|
||||
"""
|
||||
Get absolute path to a test resource file.
|
||||
|
||||
Works both when tests run from unit_tests/ directory and from connector root.
|
||||
|
||||
Args:
|
||||
resource_file: Relative path like "http/response/profiles.json"
|
||||
|
||||
Returns:
|
||||
Absolute path to the resource file
|
||||
"""
|
||||
local_path = Path("resource") / resource_file
|
||||
if local_path.exists():
|
||||
return local_path
|
||||
|
||||
connector_root_path = Path(__file__).parent / "resource" / resource_file
|
||||
if connector_root_path.exists():
|
||||
return connector_root_path
|
||||
|
||||
return local_path
|
||||
|
||||
|
||||
def get_source(config, state=None) -> YamlDeclarativeSource:
|
||||
"""
|
||||
Create a YamlDeclarativeSource instance for testing.
|
||||
|
||||
This is the main entry point for running your connector in tests.
|
||||
"""
|
||||
catalog = CatalogBuilder().build()
|
||||
state = StateBuilder().build() if not state else state
|
||||
return YamlDeclarativeSource(path_to_yaml=str(_YAML_FILE_PATH), catalog=catalog, config=config, state=state)
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def clear_cache_before_each_test():
|
||||
"""
|
||||
CRITICAL: Clear request cache before each test!
|
||||
|
||||
Without this, cached responses from one test will affect other tests,
|
||||
causing flaky, unpredictable behavior.
|
||||
"""
|
||||
cache_dir = Path(os.getenv("REQUEST_CACHE_PATH"))
|
||||
if cache_dir.exists() and cache_dir.is_dir():
|
||||
for file_path in cache_dir.glob("*.sqlite"):
|
||||
file_path.unlink()
|
||||
yield
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
@@ -0,0 +1,62 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
class ConfigBuilder:
|
||||
"""
|
||||
Builder for creating Klaviyo connector configurations for tests.
|
||||
|
||||
Example usage:
|
||||
config = (
|
||||
ConfigBuilder()
|
||||
.with_api_key("test_api_key")
|
||||
.with_start_date(datetime(2024, 1, 1))
|
||||
.build()
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._api_key: Optional[str] = None
|
||||
self._start_date: Optional[str] = None
|
||||
self._disable_fetching_predictive_analytics: bool = False
|
||||
self._num_workers: int = 10
|
||||
|
||||
def with_api_key(self, api_key: str) -> "ConfigBuilder":
|
||||
"""Set the Klaviyo API key."""
|
||||
self._api_key = api_key
|
||||
return self
|
||||
|
||||
def with_start_date(self, date: datetime) -> "ConfigBuilder":
|
||||
"""Set the replication start date (for incremental syncs)."""
|
||||
self._start_date = date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
return self
|
||||
|
||||
def with_start_date_str(self, date_str: str) -> "ConfigBuilder":
|
||||
"""Set the replication start date as a string."""
|
||||
self._start_date = date_str
|
||||
return self
|
||||
|
||||
def with_disable_fetching_predictive_analytics(self, disable: bool = True) -> "ConfigBuilder":
|
||||
"""Disable fetching predictive analytics for profiles stream."""
|
||||
self._disable_fetching_predictive_analytics = disable
|
||||
return self
|
||||
|
||||
def with_num_workers(self, num_workers: int) -> "ConfigBuilder":
|
||||
"""Set the number of concurrent workers."""
|
||||
self._num_workers = num_workers
|
||||
return self
|
||||
|
||||
def build(self) -> Dict[str, Any]:
|
||||
"""Build and return the configuration dictionary."""
|
||||
start_date = self._start_date or "2012-01-01T00:00:00Z"
|
||||
|
||||
config = {
|
||||
"api_key": self._api_key or "test_api_key_abc123",
|
||||
"start_date": start_date,
|
||||
"disable_fetching_predictive_analytics": self._disable_fetching_predictive_analytics,
|
||||
"num_workers": self._num_workers,
|
||||
}
|
||||
|
||||
return config
|
||||
@@ -0,0 +1,179 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
from typing import Dict, Optional
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from airbyte_cdk.test.mock_http import HttpRequest
|
||||
from airbyte_cdk.test.mock_http.request import ANY_QUERY_PARAMS
|
||||
|
||||
|
||||
class KlaviyoRequestBuilder:
|
||||
"""
|
||||
Builder for creating HTTP requests for Klaviyo API endpoints.
|
||||
|
||||
This builder helps create clean, reusable request definitions for tests
|
||||
instead of manually constructing HttpRequest objects each time.
|
||||
|
||||
Example usage:
|
||||
request = (
|
||||
KlaviyoRequestBuilder.profiles_endpoint("test_api_key")
|
||||
.with_page_size(100)
|
||||
.with_filter("greater-than(updated,2024-01-01T00:00:00+00:00)")
|
||||
.build()
|
||||
)
|
||||
"""
|
||||
|
||||
BASE_URL = "https://a.klaviyo.com/api"
|
||||
REVISION = "2024-10-15"
|
||||
|
||||
@classmethod
|
||||
def profiles_endpoint(cls, api_key: str) -> "KlaviyoRequestBuilder":
|
||||
"""Create a request builder for the /profiles endpoint."""
|
||||
return cls("profiles", api_key)
|
||||
|
||||
@classmethod
|
||||
def events_endpoint(cls, api_key: str) -> "KlaviyoRequestBuilder":
|
||||
"""Create a request builder for the /events endpoint."""
|
||||
return cls("events", api_key)
|
||||
|
||||
@classmethod
|
||||
def templates_endpoint(cls, api_key: str) -> "KlaviyoRequestBuilder":
|
||||
"""Create a request builder for the /templates endpoint (email_templates stream)."""
|
||||
return cls("templates", api_key)
|
||||
|
||||
@classmethod
|
||||
def campaigns_endpoint(cls, api_key: str) -> "KlaviyoRequestBuilder":
|
||||
"""Create a request builder for the /campaigns endpoint."""
|
||||
return cls("campaigns", api_key)
|
||||
|
||||
@classmethod
|
||||
def flows_endpoint(cls, api_key: str) -> "KlaviyoRequestBuilder":
|
||||
"""Create a request builder for the /flows endpoint."""
|
||||
return cls("flows", api_key)
|
||||
|
||||
@classmethod
|
||||
def metrics_endpoint(cls, api_key: str) -> "KlaviyoRequestBuilder":
|
||||
"""Create a request builder for the /metrics endpoint."""
|
||||
return cls("metrics", api_key)
|
||||
|
||||
@classmethod
|
||||
def lists_endpoint(cls, api_key: str) -> "KlaviyoRequestBuilder":
|
||||
"""Create a request builder for the /lists endpoint."""
|
||||
return cls("lists", api_key)
|
||||
|
||||
@classmethod
|
||||
def lists_detailed_endpoint(cls, api_key: str, list_id: str) -> "KlaviyoRequestBuilder":
|
||||
"""Create a request builder for the /lists/{list_id} endpoint."""
|
||||
return cls(f"lists/{list_id}", api_key)
|
||||
|
||||
@classmethod
|
||||
def campaign_recipient_estimations_endpoint(cls, api_key: str, campaign_id: str) -> "KlaviyoRequestBuilder":
|
||||
"""Create a request builder for the /campaign-recipient-estimations/{campaign_id} endpoint."""
|
||||
return cls(f"campaign-recipient-estimations/{campaign_id}", api_key)
|
||||
|
||||
@classmethod
|
||||
def from_url(cls, url: str, api_key: str) -> "KlaviyoRequestBuilder":
|
||||
"""
|
||||
Create a request builder from a full URL (used for pagination links).
|
||||
|
||||
Args:
|
||||
url: Full URL including query parameters
|
||||
api_key: The Klaviyo API key
|
||||
|
||||
Returns:
|
||||
KlaviyoRequestBuilder configured with the URL path and query params
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
path = parsed.path.replace("/api/", "")
|
||||
builder = cls(path, api_key)
|
||||
builder._full_url = url
|
||||
if parsed.query:
|
||||
query_params = parse_qs(parsed.query)
|
||||
builder._query_params = {k: v[0] if len(v) == 1 else v for k, v in query_params.items()}
|
||||
return builder
|
||||
|
||||
def __init__(self, resource: str, api_key: str):
|
||||
"""
|
||||
Initialize the request builder.
|
||||
|
||||
Args:
|
||||
resource: The API resource (e.g., 'profiles', 'events')
|
||||
api_key: The Klaviyo API key
|
||||
"""
|
||||
self._resource = resource
|
||||
self._api_key = api_key
|
||||
self._query_params: Dict = {}
|
||||
self._full_url: Optional[str] = None
|
||||
|
||||
def with_any_query_params(self) -> "KlaviyoRequestBuilder":
|
||||
"""Accept any query parameters (useful for flexible matching)."""
|
||||
self._query_params = ANY_QUERY_PARAMS
|
||||
return self
|
||||
|
||||
def with_query_params(self, query_params: dict) -> "KlaviyoRequestBuilder":
|
||||
"""Set specific query parameters for the request."""
|
||||
self._query_params = query_params
|
||||
return self
|
||||
|
||||
def with_page_size(self, size: int) -> "KlaviyoRequestBuilder":
|
||||
"""Set the page size parameter."""
|
||||
self._query_params["page[size]"] = str(size)
|
||||
return self
|
||||
|
||||
def with_filter(self, filter_expr: str) -> "KlaviyoRequestBuilder":
|
||||
"""Set the filter parameter."""
|
||||
self._query_params["filter"] = filter_expr
|
||||
return self
|
||||
|
||||
def with_sort(self, sort_field: str) -> "KlaviyoRequestBuilder":
|
||||
"""Set the sort parameter."""
|
||||
self._query_params["sort"] = sort_field
|
||||
return self
|
||||
|
||||
def with_additional_fields(self, fields: str) -> "KlaviyoRequestBuilder":
|
||||
"""Set the additional-fields[profile] parameter."""
|
||||
self._query_params["additional-fields[profile]"] = fields
|
||||
return self
|
||||
|
||||
def with_additional_fields_list(self, fields: str) -> "KlaviyoRequestBuilder":
|
||||
"""Set the additional-fields[list] parameter."""
|
||||
self._query_params["additional-fields[list]"] = fields
|
||||
return self
|
||||
|
||||
def with_fields_event(self, fields: str) -> "KlaviyoRequestBuilder":
|
||||
"""Set the fields[event] parameter."""
|
||||
self._query_params["fields[event]"] = fields
|
||||
return self
|
||||
|
||||
def with_fields_metric(self, fields: str) -> "KlaviyoRequestBuilder":
|
||||
"""Set the fields[metric] parameter."""
|
||||
self._query_params["fields[metric]"] = fields
|
||||
return self
|
||||
|
||||
def with_include(self, include: str) -> "KlaviyoRequestBuilder":
|
||||
"""Set the include parameter."""
|
||||
self._query_params["include"] = include
|
||||
return self
|
||||
|
||||
def build(self) -> HttpRequest:
|
||||
"""
|
||||
Build and return the HttpRequest object.
|
||||
|
||||
Returns:
|
||||
HttpRequest configured with the URL, query params, and headers
|
||||
"""
|
||||
if self._full_url:
|
||||
parsed = urlparse(self._full_url)
|
||||
url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
|
||||
else:
|
||||
url = f"{self.BASE_URL}/{self._resource}"
|
||||
|
||||
return HttpRequest(
|
||||
url=url,
|
||||
query_params=self._query_params if self._query_params else ANY_QUERY_PARAMS,
|
||||
headers={
|
||||
"Authorization": f"Klaviyo-API-Key {self._api_key}",
|
||||
"Accept": "application/json",
|
||||
"Revision": self.REVISION,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,158 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from airbyte_cdk.test.mock_http import HttpResponse
|
||||
from airbyte_cdk.test.mock_http.response_builder import find_template
|
||||
|
||||
|
||||
class KlaviyoPaginatedResponseBuilder:
|
||||
"""
|
||||
Builder for creating paginated Klaviyo API responses.
|
||||
|
||||
This builder simplifies creating mock responses for pagination tests by handling
|
||||
the boilerplate JSON structure that Klaviyo API returns.
|
||||
|
||||
Example usage:
|
||||
response = (
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records([record1, record2])
|
||||
.with_next_page_link("https://a.klaviyo.com/api/profiles?page[cursor]=abc123")
|
||||
.build()
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(self, base_url: str = "https://a.klaviyo.com/api"):
|
||||
"""
|
||||
Initialize the response builder.
|
||||
|
||||
Args:
|
||||
base_url: Base URL for the API (default: Klaviyo API)
|
||||
"""
|
||||
self.base_url = base_url
|
||||
self.records: List[Dict[str, Any]] = []
|
||||
self._next_page_link: Optional[str] = None
|
||||
self._self_link: Optional[str] = None
|
||||
|
||||
def with_records(self, records: List[Dict[str, Any]]) -> "KlaviyoPaginatedResponseBuilder":
|
||||
"""
|
||||
Add records to the response.
|
||||
|
||||
Args:
|
||||
records: List of record dictionaries to include in the response
|
||||
|
||||
Returns:
|
||||
Self for method chaining
|
||||
"""
|
||||
self.records = records
|
||||
return self
|
||||
|
||||
def with_next_page_link(self, next_link: str) -> "KlaviyoPaginatedResponseBuilder":
|
||||
"""
|
||||
Set the next page link for pagination.
|
||||
|
||||
Args:
|
||||
next_link: Full URL for the next page
|
||||
|
||||
Returns:
|
||||
Self for method chaining
|
||||
"""
|
||||
self._next_page_link = next_link
|
||||
return self
|
||||
|
||||
def with_self_link(self, self_link: str) -> "KlaviyoPaginatedResponseBuilder":
|
||||
"""
|
||||
Set the self link for the current page.
|
||||
|
||||
Args:
|
||||
self_link: Full URL for the current page
|
||||
|
||||
Returns:
|
||||
Self for method chaining
|
||||
"""
|
||||
self._self_link = self_link
|
||||
return self
|
||||
|
||||
def build(self) -> HttpResponse:
|
||||
"""
|
||||
Build the HTTP response with paginated data.
|
||||
|
||||
Returns:
|
||||
HttpResponse object with the paginated response body
|
||||
"""
|
||||
links: Dict[str, Optional[str]] = {}
|
||||
|
||||
if self._self_link:
|
||||
links["self"] = self._self_link
|
||||
|
||||
if self._next_page_link:
|
||||
links["next"] = self._next_page_link
|
||||
|
||||
response_body: Dict[str, Any] = {
|
||||
"data": self.records,
|
||||
}
|
||||
|
||||
if links:
|
||||
response_body["links"] = links
|
||||
|
||||
return HttpResponse(body=json.dumps(response_body), status_code=200)
|
||||
|
||||
@classmethod
|
||||
def single_page(cls, records: List[Dict[str, Any]]) -> HttpResponse:
|
||||
"""
|
||||
Convenience method to create a single-page response.
|
||||
|
||||
Args:
|
||||
records: List of records to include
|
||||
|
||||
Returns:
|
||||
HttpResponse for a single page with no pagination links
|
||||
"""
|
||||
return cls().with_records(records).build()
|
||||
|
||||
@classmethod
|
||||
def empty_page(cls) -> HttpResponse:
|
||||
"""
|
||||
Convenience method to create an empty response.
|
||||
|
||||
Returns:
|
||||
HttpResponse for an empty result set
|
||||
"""
|
||||
return cls().with_records([]).build()
|
||||
|
||||
|
||||
def create_response(resource_name: str, status_code: int = 200, has_next: bool = False, next_cursor: Optional[str] = None) -> HttpResponse:
|
||||
"""
|
||||
Create HTTP response using template from resource/http/response/<resource_name>.json
|
||||
|
||||
Args:
|
||||
resource_name: Name of the JSON file (without .json extension)
|
||||
status_code: HTTP status code
|
||||
has_next: Whether there's a next page (for pagination)
|
||||
next_cursor: Cursor value for pagination
|
||||
"""
|
||||
body = json.dumps(find_template(resource_name, __file__))
|
||||
|
||||
return HttpResponse(body, status_code)
|
||||
|
||||
|
||||
def error_response(status_code: int, error_message: str = "Error occurred") -> HttpResponse:
|
||||
"""Create error response (401, 403, 429, etc.)"""
|
||||
error_body = {
|
||||
"errors": [
|
||||
{
|
||||
"id": "error-id",
|
||||
"status": status_code,
|
||||
"code": "error_code",
|
||||
"title": "Error",
|
||||
"detail": error_message,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
headers = {}
|
||||
if status_code == 429:
|
||||
headers["Retry-After"] = "1"
|
||||
|
||||
return HttpResponse(json.dumps(error_body), status_code, headers)
|
||||
@@ -0,0 +1,550 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest import TestCase
|
||||
|
||||
import freezegun
|
||||
from unit_tests.conftest import get_source
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from airbyte_cdk.test.catalog_builder import CatalogBuilder
|
||||
from airbyte_cdk.test.entrypoint_wrapper import read
|
||||
from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse
|
||||
from airbyte_cdk.test.state_builder import StateBuilder
|
||||
from integration.config import ConfigBuilder
|
||||
from integration.request_builder import KlaviyoRequestBuilder
|
||||
from integration.response_builder import KlaviyoPaginatedResponseBuilder
|
||||
|
||||
|
||||
_NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
_STREAM_NAME = "campaigns"
|
||||
_API_KEY = "test_api_key_abc123"
|
||||
|
||||
|
||||
@freezegun.freeze_time(_NOW.isoformat())
|
||||
class TestCampaignsStream(TestCase):
|
||||
"""
|
||||
Tests for the Klaviyo 'campaigns' stream.
|
||||
|
||||
Stream configuration from manifest.yaml:
|
||||
- Uses ListPartitionRouter to iterate over campaign statuses (draft, scheduled, sent, cancelled)
|
||||
- Incremental sync with DatetimeBasedCursor on 'updated_at' field
|
||||
- Pagination: CursorPagination
|
||||
- Error handling: 429 RATE_LIMITED, 401/403 FAIL
|
||||
- Transformations: AddFields to extract 'updated_at' from attributes
|
||||
"""
|
||||
|
||||
@HttpMocker()
|
||||
def test_full_refresh_single_page(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test full refresh sync with a single page of results.
|
||||
|
||||
Given: A configured Klaviyo connector
|
||||
When: Running a full refresh sync for the campaigns stream
|
||||
Then: The connector should make requests for each campaign partition (campaign_type x archived)
|
||||
|
||||
Note: The campaigns stream uses two ListPartitionRouters:
|
||||
- campaign_type: ["sms", "email"]
|
||||
- archived: ["true", "false"]
|
||||
This creates 4 partitions total (2x2).
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock all 4 partitions (campaign_type x archived): sms/true, sms/false, email/true, email/false
|
||||
# Each partition has a specific filter value
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))",
|
||||
"sort": "updated_at",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_001",
|
||||
"attributes": {
|
||||
"name": "Test Campaign",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T10:00:00+00:00",
|
||||
"updated_at": "2024-05-31T12:30:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 4
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "campaign_001" in record_ids
|
||||
|
||||
@HttpMocker()
|
||||
def test_partition_router_multiple_statuses(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the ListPartitionRouter correctly iterates over all campaign statuses.
|
||||
|
||||
The manifest configures:
|
||||
partition_router:
|
||||
type: ListPartitionRouter
|
||||
values: ["draft", "scheduled", "sent", "cancelled"]
|
||||
cursor_field: "status"
|
||||
|
||||
Given: An API that returns campaigns for each status
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should make requests for each status partition
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock all 4 partitions (campaign_type x archived): sms/true, sms/false, email/true, email/false
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))",
|
||||
"sort": "updated_at",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_001",
|
||||
"attributes": {
|
||||
"name": "Test Campaign",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T10:00:00+00:00",
|
||||
"updated_at": "2024-05-31T12:30:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 4
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "campaign_001" in record_ids
|
||||
|
||||
@HttpMocker()
|
||||
def test_pagination_multiple_pages(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fetches all pages when pagination is present.
|
||||
|
||||
Given: An API that returns multiple pages of campaigns
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should follow pagination links and return all records
|
||||
|
||||
Note: Uses with_any_query_params() because pagination adds page[cursor] to the
|
||||
request params, making exact matching impractical. Partition behavior is tested
|
||||
separately in test_partition_router_multiple_statuses.
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Use a single mock with any query params since pagination adds page[cursor]
|
||||
# which makes exact query param matching impractical
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
[
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_001",
|
||||
"attributes": {
|
||||
"name": "Campaign 1",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T10:00:00+00:00",
|
||||
"updated_at": "2024-05-31T10:00:00+00:00",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.with_next_page_link("https://a.klaviyo.com/api/campaigns?page[cursor]=abc123")
|
||||
.build(),
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_002",
|
||||
"attributes": {
|
||||
"name": "Campaign 2",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T11:00:00+00:00",
|
||||
"updated_at": "2024-05-31T11:00:00+00:00",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.build(),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
# Using >= because with_any_query_params() matches all 4 ListPartitionRouter partitions,
|
||||
# and the mock response sequence is shared across partitions, making exact count non-deterministic.
|
||||
assert len(output.records) >= 2
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "campaign_001" in record_ids or "campaign_002" in record_ids
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_first_sync_no_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test first incremental sync with no previous state.
|
||||
|
||||
Given: No previous state (first sync)
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use start_date from config and emit state message
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock all 4 partitions (campaign_type x archived)
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))",
|
||||
"sort": "updated_at",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_001",
|
||||
"attributes": {
|
||||
"name": "Test Campaign",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T10:00:00+00:00",
|
||||
"updated_at": "2024-05-31T12:30:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 4
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "campaign_001" in record_ids
|
||||
assert len(output.state_messages) > 0
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_with_prior_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test incremental sync with a prior state from previous sync.
|
||||
|
||||
Given: A previous sync state with an updated_at cursor value
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use the state cursor and return only new/updated records
|
||||
|
||||
Note: Uses with_any_query_params() because the state cursor value affects the filter
|
||||
dynamically, making exact matching impractical.
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated_at": "2024-03-01T00:00:00+00:00"}).build()
|
||||
|
||||
# Use a single mock with any query params since state cursor affects the filter
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_new",
|
||||
"attributes": {
|
||||
"name": "New Campaign",
|
||||
"status": "sent",
|
||||
"created_at": "2024-03-10T10:00:00+00:00",
|
||||
"updated_at": "2024-03-15T10:00:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
# Using >= because with_any_query_params() matches all 4 ListPartitionRouter partitions,
|
||||
# and the mock response is shared across partitions, making exact count non-deterministic.
|
||||
assert len(output.records) >= 1
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "campaign_new" in record_ids
|
||||
assert len(output.state_messages) > 0
|
||||
|
||||
@HttpMocker()
|
||||
def test_transformation_adds_updated_at_field(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the AddFields transformation correctly extracts 'updated_at' from attributes.
|
||||
|
||||
Given: A campaign record with updated_at in attributes
|
||||
When: Running a sync
|
||||
Then: The 'updated_at' field should be added at the root level of the record
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock all 4 partitions (campaign_type x archived)
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))",
|
||||
"sort": "updated_at",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_transform_test",
|
||||
"attributes": {
|
||||
"name": "Transform Test",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T10:00:00+00:00",
|
||||
"updated_at": "2024-05-31T14:45:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 4
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "campaign_transform_test" in record_ids
|
||||
record = output.records[0].record.data
|
||||
assert "updated_at" in record
|
||||
assert record["updated_at"] == "2024-05-31T14:45:00+00:00"
|
||||
|
||||
@HttpMocker()
|
||||
def test_rate_limit_429_handling(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles 429 rate limit responses with RATE_LIMITED action.
|
||||
|
||||
Given: An API that returns a 429 rate limit error
|
||||
When: Making an API request
|
||||
Then: The connector should respect the Retry-After header and retry
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock all 4 partitions with rate limit handling
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))",
|
||||
"sort": "updated_at",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
[
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Rate limit exceeded"}]}),
|
||||
status_code=429,
|
||||
headers={"Retry-After": "1"},
|
||||
),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_after_retry",
|
||||
"attributes": {
|
||||
"name": "After Retry",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T10:00:00+00:00",
|
||||
"updated_at": "2024-05-31T10:00:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 4
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "campaign_after_retry" in record_ids
|
||||
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
# Check for backoff log message pattern
|
||||
assert any(
|
||||
"Backing off" in msg and "UserDefinedBackoffException" in msg and "429" in msg for msg in log_messages
|
||||
), "Expected backoff log message for 429 rate limit"
|
||||
# Check for retry/sleeping log message pattern
|
||||
assert any(
|
||||
"Sleeping for" in msg and "seconds" in msg for msg in log_messages
|
||||
), "Expected retry sleeping log message for 429 rate limit"
|
||||
|
||||
@HttpMocker()
|
||||
def test_unauthorized_401_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 401 Unauthorized errors with FAIL action.
|
||||
|
||||
Given: Invalid API credentials
|
||||
When: Making an API request that returns 401
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key("invalid_key").with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint("invalid_key").with_any_query_params().build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Invalid API key"}]}),
|
||||
status_code=401,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 401 authentication failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_forbidden_403_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 403 Forbidden errors with FAIL action.
|
||||
|
||||
The manifest configures 403 errors with action: FAIL, which means the connector
|
||||
should fail the sync when permission errors occur.
|
||||
|
||||
Given: API credentials with insufficient permissions
|
||||
When: Making an API request that returns 403
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Forbidden - insufficient permissions"}]}),
|
||||
status_code=403,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 403 permission failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_empty_results(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles empty results gracefully.
|
||||
|
||||
Given: An API that returns no campaigns
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should return zero records without errors
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock all 4 partitions with empty results
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))",
|
||||
"sort": "updated_at",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"data": [], "links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None}}),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 0
|
||||
assert not any(log.log.level == "ERROR" for log in output.logs)
|
||||
@@ -0,0 +1,600 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest import TestCase
|
||||
|
||||
import freezegun
|
||||
from unit_tests.conftest import get_source
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from airbyte_cdk.test.catalog_builder import CatalogBuilder
|
||||
from airbyte_cdk.test.entrypoint_wrapper import read
|
||||
from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse
|
||||
from airbyte_cdk.test.state_builder import StateBuilder
|
||||
from integration.config import ConfigBuilder
|
||||
from integration.request_builder import KlaviyoRequestBuilder
|
||||
from integration.response_builder import KlaviyoPaginatedResponseBuilder
|
||||
|
||||
|
||||
_NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
_STREAM_NAME = "campaigns_detailed"
|
||||
_API_KEY = "test_api_key_abc123"
|
||||
|
||||
|
||||
@freezegun.freeze_time(_NOW.isoformat())
|
||||
class TestCampaignsDetailedStream(TestCase):
|
||||
"""
|
||||
Tests for the Klaviyo 'campaigns_detailed' stream.
|
||||
|
||||
Stream configuration from manifest.yaml:
|
||||
- Uses CustomTransformation to flatten campaign message data
|
||||
- Uses ListPartitionRouter to iterate over campaign statuses
|
||||
- Incremental sync with DatetimeBasedCursor on 'updated_at' field
|
||||
- Request parameters: include=campaign-messages
|
||||
- Pagination: CursorPagination
|
||||
- Error handling: 429 RATE_LIMITED, 401/403 FAIL
|
||||
"""
|
||||
|
||||
@HttpMocker()
|
||||
def test_full_refresh_with_included_messages(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test full refresh sync with included campaign message data.
|
||||
|
||||
The CustomTransformation flattens the included campaign-messages data into each campaign record.
|
||||
|
||||
Given: An API response with campaigns and included campaign-messages
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should return campaigns with message data merged in
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# campaigns_detailed uses ListPartitionRouter with 4 partitions (campaign_type: sms/email × archived: true/false)
|
||||
# Mock all 4 partition combinations with explicit query params
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
filter_value = f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))"
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY)
|
||||
.with_query_params({"filter": filter_value, "sort": "updated_at"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_001",
|
||||
"attributes": {
|
||||
"name": "Test Campaign",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T10:00:00+00:00",
|
||||
"updated_at": "2024-05-31T12:30:00+00:00",
|
||||
"send_time": "2024-05-31T10:00:00+00:00",
|
||||
},
|
||||
"relationships": {
|
||||
"campaign-messages": {"data": [{"type": "campaign-message", "id": "msg_001"}]},
|
||||
},
|
||||
}
|
||||
],
|
||||
"included": [
|
||||
{
|
||||
"type": "campaign-message",
|
||||
"id": "msg_001",
|
||||
"attributes": {
|
||||
"label": "Email Message",
|
||||
"channel": "email",
|
||||
"content": {"subject": "Welcome!", "preview_text": "Thanks for joining"},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
# Mock the campaign-recipient-estimations endpoint (called by CustomTransformation)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaign_recipient_estimations_endpoint(_API_KEY, "campaign_001").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "campaign-recipient-estimation",
|
||||
"id": "campaign_001",
|
||||
"attributes": {"estimated_recipient_count": 1000},
|
||||
}
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 4
|
||||
record = output.records[0].record.data
|
||||
assert record["id"] == "campaign_001"
|
||||
assert record["attributes"]["name"] == "Test Campaign"
|
||||
|
||||
@HttpMocker()
|
||||
def test_pagination_multiple_pages(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fetches all pages when pagination is present.
|
||||
|
||||
Given: An API that returns multiple pages of campaigns with included messages
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should follow pagination links and return all records
|
||||
|
||||
Note: Uses with_any_query_params() because pagination adds page[cursor] to the
|
||||
request params, making exact matching impractical.
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Use a single mock with any query params since pagination adds page[cursor]
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
[
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_001",
|
||||
"attributes": {
|
||||
"name": "Campaign 1",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T10:00:00+00:00",
|
||||
"updated_at": "2024-05-31T10:00:00+00:00",
|
||||
},
|
||||
"relationships": {"campaign-messages": {"data": []}},
|
||||
}
|
||||
]
|
||||
)
|
||||
.with_next_page_link("https://a.klaviyo.com/api/campaigns?page[cursor]=abc123")
|
||||
.build(),
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_002",
|
||||
"attributes": {
|
||||
"name": "Campaign 2",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T11:00:00+00:00",
|
||||
"updated_at": "2024-05-31T11:00:00+00:00",
|
||||
},
|
||||
"relationships": {"campaign-messages": {"data": []}},
|
||||
}
|
||||
]
|
||||
)
|
||||
.build(),
|
||||
],
|
||||
)
|
||||
|
||||
# Mock the campaign-recipient-estimations endpoint for both campaigns
|
||||
for campaign_id in ["campaign_001", "campaign_002"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaign_recipient_estimations_endpoint(_API_KEY, campaign_id).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "campaign-recipient-estimation",
|
||||
"id": campaign_id,
|
||||
"attributes": {"estimated_recipient_count": 1000},
|
||||
}
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 5
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "campaign_001" in record_ids and "campaign_002" in record_ids
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_first_sync_no_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test first incremental sync with no previous state.
|
||||
|
||||
Given: No previous state (first sync)
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use start_date from config and emit state message
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# campaigns_detailed uses ListPartitionRouter with 4 partitions (campaign_type: sms/email × archived: true/false)
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
filter_value = f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))"
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY)
|
||||
.with_query_params({"filter": filter_value, "sort": "updated_at"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_001",
|
||||
"attributes": {
|
||||
"name": "Test Campaign",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T10:00:00+00:00",
|
||||
"updated_at": "2024-05-31T12:30:00+00:00",
|
||||
},
|
||||
"relationships": {"campaign-messages": {"data": []}},
|
||||
}
|
||||
],
|
||||
"included": [],
|
||||
"links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
# Mock the campaign-recipient-estimations endpoint (called by CustomTransformation)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaign_recipient_estimations_endpoint(_API_KEY, "campaign_001").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "campaign-recipient-estimation",
|
||||
"id": "campaign_001",
|
||||
"attributes": {"estimated_recipient_count": 1000},
|
||||
}
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 4
|
||||
assert len(output.state_messages) > 0
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_with_prior_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test incremental sync with a prior state from previous sync.
|
||||
|
||||
Given: A previous sync state with an updated_at cursor value
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use the state cursor and return only new/updated records
|
||||
|
||||
Note: Uses with_any_query_params() because the state cursor value affects the filter
|
||||
dynamically, making exact matching impractical.
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated_at": "2024-03-01T00:00:00+00:00"}).build()
|
||||
|
||||
# Use a single mock with any query params since state cursor affects the filter
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_new",
|
||||
"attributes": {
|
||||
"name": "New Campaign",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T10:00:00+00:00",
|
||||
"updated_at": "2024-05-31T10:00:00+00:00",
|
||||
},
|
||||
"relationships": {"campaign-messages": {"data": []}},
|
||||
}
|
||||
],
|
||||
"included": [],
|
||||
"links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
# Mock the campaign-recipient-estimations endpoint (called by CustomTransformation)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaign_recipient_estimations_endpoint(_API_KEY, "campaign_new").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "campaign-recipient-estimation",
|
||||
"id": "campaign_new",
|
||||
"attributes": {"estimated_recipient_count": 1000},
|
||||
}
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
assert len(output.records) == 4
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "campaign_new" in record_ids
|
||||
assert len(output.state_messages) > 0
|
||||
|
||||
@HttpMocker()
|
||||
def test_transformation_adds_updated_at_field(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the AddFields transformation correctly extracts 'updated_at' from attributes.
|
||||
|
||||
Given: A campaign record with updated_at in attributes
|
||||
When: Running a sync
|
||||
Then: The 'updated_at' field should be added at the root level of the record
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# campaigns_detailed uses ListPartitionRouter with 4 partitions (campaign_type: sms/email × archived: true/false)
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
filter_value = f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))"
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY)
|
||||
.with_query_params({"filter": filter_value, "sort": "updated_at"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_transform_test",
|
||||
"attributes": {
|
||||
"name": "Transform Test",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T10:00:00+00:00",
|
||||
"updated_at": "2024-05-31T14:45:00+00:00",
|
||||
},
|
||||
"relationships": {"campaign-messages": {"data": []}},
|
||||
}
|
||||
],
|
||||
"included": [],
|
||||
"links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
# Mock the campaign-recipient-estimations endpoint (called by CustomTransformation)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaign_recipient_estimations_endpoint(_API_KEY, "campaign_transform_test").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "campaign-recipient-estimation",
|
||||
"id": "campaign_transform_test",
|
||||
"attributes": {"estimated_recipient_count": 1000},
|
||||
}
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 4
|
||||
record = output.records[0].record.data
|
||||
assert "updated_at" in record
|
||||
assert record["updated_at"] == "2024-05-31T14:45:00+00:00"
|
||||
|
||||
@HttpMocker()
|
||||
def test_rate_limit_429_handling(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles 429 rate limit responses with RATE_LIMITED action.
|
||||
|
||||
Given: An API that returns a 429 rate limit error
|
||||
When: Making an API request
|
||||
Then: The connector should respect the Retry-After header and retry
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# campaigns_detailed uses ListPartitionRouter with 4 partitions (campaign_type: sms/email × archived: true/false)
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
filter_value = f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))"
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY)
|
||||
.with_query_params({"filter": filter_value, "sort": "updated_at"})
|
||||
.build(),
|
||||
[
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Rate limit exceeded"}]}),
|
||||
status_code=429,
|
||||
headers={"Retry-After": "1"},
|
||||
),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_after_retry",
|
||||
"attributes": {
|
||||
"name": "After Retry",
|
||||
"status": "sent",
|
||||
"created_at": "2024-05-31T10:00:00+00:00",
|
||||
"updated_at": "2024-05-31T10:00:00+00:00",
|
||||
},
|
||||
"relationships": {"campaign-messages": {"data": []}},
|
||||
}
|
||||
],
|
||||
"included": [],
|
||||
"links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# Mock the campaign-recipient-estimations endpoint (called by CustomTransformation)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaign_recipient_estimations_endpoint(_API_KEY, "campaign_after_retry").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "campaign-recipient-estimation",
|
||||
"id": "campaign_after_retry",
|
||||
"attributes": {"estimated_recipient_count": 1000},
|
||||
}
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 4
|
||||
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
# Check for backoff log message pattern
|
||||
assert any(
|
||||
"Backing off" in msg and "UserDefinedBackoffException" in msg and "429" in msg for msg in log_messages
|
||||
), "Expected backoff log message for 429 rate limit"
|
||||
# Check for retry/sleeping log message pattern
|
||||
assert any(
|
||||
"Sleeping for" in msg and "seconds" in msg for msg in log_messages
|
||||
), "Expected retry sleeping log message for 429 rate limit"
|
||||
|
||||
@HttpMocker()
|
||||
def test_unauthorized_401_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 401 Unauthorized errors with FAIL action.
|
||||
|
||||
Given: Invalid API credentials
|
||||
When: Making an API request that returns 401
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key("invalid_key").with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# campaigns_detailed uses ListPartitionRouter with 4 partitions (campaign_type: sms/email × archived: true/false)
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
filter_value = f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))"
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint("invalid_key")
|
||||
.with_query_params({"filter": filter_value, "sort": "updated_at"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Invalid API key"}]}),
|
||||
status_code=401,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 401 authentication failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_forbidden_403_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 403 Forbidden errors with FAIL action.
|
||||
|
||||
The manifest configures 403 errors with action: FAIL, which means the connector
|
||||
should fail the sync when permission errors occur.
|
||||
|
||||
Given: API credentials with insufficient permissions
|
||||
When: Making an API request that returns 403
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# campaigns_detailed uses ListPartitionRouter with 4 partitions (campaign_type: sms/email × archived: true/false)
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
filter_value = f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))"
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY)
|
||||
.with_query_params({"filter": filter_value, "sort": "updated_at"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Forbidden - insufficient permissions"}]}),
|
||||
status_code=403,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 403 permission failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_empty_results(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles empty results gracefully.
|
||||
|
||||
Given: An API that returns no campaigns
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should return zero records without errors
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# campaigns_detailed uses ListPartitionRouter with 4 partitions (campaign_type: sms/email × archived: true/false)
|
||||
for campaign_type in ["sms", "email"]:
|
||||
for archived in ["true", "false"]:
|
||||
filter_value = f"and(greater-or-equal(updated_at,2024-05-31T00:00:00+0000),less-or-equal(updated_at,2024-06-01T12:00:00+0000),equals(messages.channel,'{campaign_type}'),equals(archived,{archived}))"
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.campaigns_endpoint(_API_KEY)
|
||||
.with_query_params({"filter": filter_value, "sort": "updated_at"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{"data": [], "included": [], "links": {"self": "https://a.klaviyo.com/api/campaigns", "next": None}}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 0
|
||||
assert not any(log.log.level == "ERROR" for log in output.logs)
|
||||
@@ -0,0 +1,486 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest import TestCase
|
||||
|
||||
import freezegun
|
||||
from unit_tests.conftest import get_source
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from airbyte_cdk.test.catalog_builder import CatalogBuilder
|
||||
from airbyte_cdk.test.entrypoint_wrapper import read
|
||||
from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse
|
||||
from airbyte_cdk.test.state_builder import StateBuilder
|
||||
from integration.config import ConfigBuilder
|
||||
from integration.request_builder import KlaviyoRequestBuilder
|
||||
from integration.response_builder import KlaviyoPaginatedResponseBuilder
|
||||
|
||||
|
||||
_NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
_STREAM_NAME = "email_templates"
|
||||
_API_KEY = "test_api_key_abc123"
|
||||
|
||||
|
||||
@freezegun.freeze_time(_NOW.isoformat())
|
||||
class TestEmailTemplatesStream(TestCase):
|
||||
"""
|
||||
Tests for the Klaviyo 'email_templates' stream.
|
||||
|
||||
Stream configuration from manifest.yaml:
|
||||
- Incremental sync with DatetimeBasedCursor on 'updated' field
|
||||
- Pagination: CursorPagination with page[size]=100
|
||||
- Error handling: 429 RATE_LIMITED, 401/403 FAIL
|
||||
- Transformations: AddFields to extract 'updated' from attributes
|
||||
"""
|
||||
|
||||
@HttpMocker()
|
||||
def test_full_refresh_single_page(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test full refresh sync with a single page of results.
|
||||
|
||||
Given: A configured Klaviyo connector
|
||||
When: Running a full refresh sync for the email_templates stream
|
||||
Then: The connector should make the correct API request and return all records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.templates_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "template",
|
||||
"id": "template_001",
|
||||
"attributes": {
|
||||
"name": "Welcome Email",
|
||||
"editor_type": "CODE",
|
||||
"html": "<html><body>Welcome!</body></html>",
|
||||
"text": "Welcome!",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/templates", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert record["id"] == "template_001"
|
||||
assert record["attributes"]["name"] == "Welcome Email"
|
||||
|
||||
@HttpMocker()
|
||||
def test_pagination_multiple_pages(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fetches all pages when pagination is present.
|
||||
|
||||
Note: This test also validates pagination behavior for other streams using the same
|
||||
CursorPagination pattern (profiles, events, flows, metrics, lists).
|
||||
|
||||
Given: An API that returns multiple pages of templates
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should follow pagination links and return all records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Use a single mock with multiple responses to avoid ambiguity in mock matching.
|
||||
# The first response includes a next_page_link, the second response has no next link.
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.templates_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
[
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "template",
|
||||
"id": "template_001",
|
||||
"attributes": {
|
||||
"name": "Template 1",
|
||||
"editor_type": "CODE",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.with_next_page_link("https://a.klaviyo.com/api/templates?page[cursor]=abc123")
|
||||
.build(),
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "template",
|
||||
"id": "template_002",
|
||||
"attributes": {
|
||||
"name": "Template 2",
|
||||
"editor_type": "DRAG_AND_DROP",
|
||||
"created": "2024-05-31T11:00:00+00:00",
|
||||
"updated": "2024-05-31T11:00:00+00:00",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.build(),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 2
|
||||
assert output.records[0].record.data["id"] == "template_001"
|
||||
assert output.records[1].record.data["id"] == "template_002"
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_first_sync_no_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test first incremental sync with no previous state.
|
||||
|
||||
Given: No previous state (first sync)
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use start_date from config and emit state message
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.templates_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "template",
|
||||
"id": "template_001",
|
||||
"attributes": {
|
||||
"name": "Welcome Email",
|
||||
"editor_type": "CODE",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/templates", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "template_001"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
latest_state = output.most_recent_state.stream_state.__dict__
|
||||
assert "updated" in latest_state
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_with_prior_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test incremental sync with a prior state from previous sync.
|
||||
|
||||
Given: A previous sync state with an updated cursor value
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use the state cursor and return only new/updated records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": "2024-05-31T00:00:00+00:00"}).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.templates_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "template",
|
||||
"id": "template_new",
|
||||
"attributes": {
|
||||
"name": "New Template",
|
||||
"editor_type": "CODE",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/templates", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "template_new"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
latest_state = output.most_recent_state.stream_state.__dict__
|
||||
# Note: The connector returns datetime with +0000 format (without colon)
|
||||
assert latest_state["updated"] == "2024-05-31T10:00:00+0000"
|
||||
|
||||
@HttpMocker()
|
||||
def test_transformation_adds_updated_field(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the AddFields transformation correctly extracts 'updated' from attributes.
|
||||
|
||||
Given: A template record with updated in attributes
|
||||
When: Running a sync
|
||||
Then: The 'updated' field should be added at the root level of the record
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.templates_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "template",
|
||||
"id": "template_transform_test",
|
||||
"attributes": {
|
||||
"name": "Transform Test",
|
||||
"editor_type": "CODE",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T14:45:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/templates", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert "updated" in record
|
||||
assert record["updated"] == "2024-05-31T14:45:00+00:00"
|
||||
|
||||
@HttpMocker()
|
||||
def test_rate_limit_429_handling(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles 429 rate limit responses with RATE_LIMITED action.
|
||||
|
||||
Given: An API that returns a 429 rate limit error
|
||||
When: Making an API request
|
||||
Then: The connector should respect the Retry-After header and retry
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.templates_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
[
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Rate limit exceeded"}]}),
|
||||
status_code=429,
|
||||
headers={"Retry-After": "1"},
|
||||
),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "template",
|
||||
"id": "template_after_retry",
|
||||
"attributes": {
|
||||
"name": "After Retry",
|
||||
"editor_type": "CODE",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/templates", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "template_after_retry"
|
||||
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
# Check for backoff log message pattern
|
||||
assert any(
|
||||
"Backing off" in msg and "UserDefinedBackoffException" in msg and "429" in msg for msg in log_messages
|
||||
), "Expected backoff log message for 429 rate limit"
|
||||
# Check for retry/sleeping log message pattern
|
||||
assert any(
|
||||
"Sleeping for" in msg and "seconds" in msg for msg in log_messages
|
||||
), "Expected retry sleeping log message for 429 rate limit"
|
||||
|
||||
@HttpMocker()
|
||||
def test_unauthorized_401_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 401 Unauthorized errors with FAIL action.
|
||||
|
||||
Given: Invalid API credentials
|
||||
When: Making an API request that returns 401
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key("invalid_key").with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.templates_endpoint("invalid_key")
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Invalid API key"}]}),
|
||||
status_code=401,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 401 authentication failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_forbidden_403_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 403 Forbidden errors with FAIL action.
|
||||
|
||||
The manifest configures 403 errors with action: FAIL, which means the connector
|
||||
should fail the sync when permission errors occur.
|
||||
|
||||
Given: API credentials with insufficient permissions
|
||||
When: Making an API request that returns 403
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.templates_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Forbidden - insufficient permissions"}]}),
|
||||
status_code=403,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 403 permission failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_empty_results(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles empty results gracefully.
|
||||
|
||||
Given: An API that returns no templates
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should return zero records without errors
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.templates_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"data": [], "links": {"self": "https://a.klaviyo.com/api/templates", "next": None}}),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 0
|
||||
assert not any(log.log.level == "ERROR" for log in output.logs)
|
||||
@@ -0,0 +1,537 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest import TestCase
|
||||
|
||||
import freezegun
|
||||
from unit_tests.conftest import get_source
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from airbyte_cdk.test.catalog_builder import CatalogBuilder
|
||||
from airbyte_cdk.test.entrypoint_wrapper import read
|
||||
from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse
|
||||
from airbyte_cdk.test.state_builder import StateBuilder
|
||||
from integration.config import ConfigBuilder
|
||||
from integration.request_builder import KlaviyoRequestBuilder
|
||||
from integration.response_builder import KlaviyoPaginatedResponseBuilder
|
||||
|
||||
|
||||
_NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
_STREAM_NAME = "events"
|
||||
_API_KEY = "test_api_key_abc123"
|
||||
|
||||
|
||||
@freezegun.freeze_time(_NOW.isoformat())
|
||||
class TestEventsStream(TestCase):
|
||||
"""
|
||||
Tests for the Klaviyo 'events' stream.
|
||||
|
||||
Stream configuration from manifest.yaml:
|
||||
- Incremental sync with DatetimeBasedCursor on 'datetime' field
|
||||
- Step: P7D (7 days) with cursor_granularity: PT1S
|
||||
- Pagination: CursorPagination
|
||||
- Error handling: 429 RATE_LIMITED, 401/403 FAIL
|
||||
- Transformations: AddFields to extract 'datetime' from attributes
|
||||
- Request parameters: fields[event], fields[metric], include, filter, sort
|
||||
"""
|
||||
|
||||
@HttpMocker()
|
||||
def test_full_refresh_single_page(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test full refresh sync with a single page of results.
|
||||
|
||||
Given: A configured Klaviyo connector
|
||||
When: Running a full refresh sync for the events stream
|
||||
Then: The connector should make the correct API request and return all records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"fields[event]": "event_properties,timestamp,uuid,datetime",
|
||||
"fields[metric]": "name,created,updated,integration",
|
||||
"include": "metric,attributions",
|
||||
"filter": "greater-or-equal(datetime,2024-05-31T00:00:00+0000),less-or-equal(datetime,2024-06-01T12:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_001",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T10:30:00+00:00",
|
||||
"datetime": "2024-05-31T10:30:00+00:00",
|
||||
"uuid": "550e8400-e29b-41d4-a716-446655440000",
|
||||
"event_properties": {"value": 99.99, "currency": "USD"},
|
||||
},
|
||||
"relationships": {
|
||||
"metric": {"data": {"type": "metric", "id": "metric_001"}},
|
||||
"attributions": {"data": []},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/events", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert record["id"] == "event_001"
|
||||
assert record["attributes"]["uuid"] == "550e8400-e29b-41d4-a716-446655440000"
|
||||
|
||||
@HttpMocker()
|
||||
def test_pagination_multiple_pages(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fetches all pages when pagination is present.
|
||||
|
||||
Given: An API that returns multiple pages of events
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should follow pagination links and return all records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Use a single mock with multiple responses to avoid ambiguity in mock matching.
|
||||
# The first response includes a next_page_link, the second response has no next link.
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
[
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_001",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T10:00:00+00:00",
|
||||
"datetime": "2024-05-31T10:00:00+00:00",
|
||||
"uuid": "uuid-001",
|
||||
"event_properties": {},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m1"}}, "attributions": {"data": []}},
|
||||
},
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_002",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T11:00:00+00:00",
|
||||
"datetime": "2024-05-31T11:00:00+00:00",
|
||||
"uuid": "uuid-002",
|
||||
"event_properties": {},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m1"}}, "attributions": {"data": []}},
|
||||
},
|
||||
]
|
||||
)
|
||||
.with_next_page_link("https://a.klaviyo.com/api/events?page[cursor]=abc123")
|
||||
.build(),
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_003",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T12:00:00+00:00",
|
||||
"datetime": "2024-05-31T12:00:00+00:00",
|
||||
"uuid": "uuid-003",
|
||||
"event_properties": {},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m1"}}, "attributions": {"data": []}},
|
||||
}
|
||||
]
|
||||
)
|
||||
.build(),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 3
|
||||
assert output.records[0].record.data["id"] == "event_001"
|
||||
assert output.records[1].record.data["id"] == "event_002"
|
||||
assert output.records[2].record.data["id"] == "event_003"
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_first_sync_no_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test first incremental sync with no previous state.
|
||||
|
||||
Given: No previous state (first sync)
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use start_date from config and emit state message
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"fields[event]": "event_properties,timestamp,uuid,datetime",
|
||||
"fields[metric]": "name,created,updated,integration",
|
||||
"include": "metric,attributions",
|
||||
"filter": "greater-or-equal(datetime,2024-05-31T00:00:00+0000),less-or-equal(datetime,2024-06-01T12:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_001",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T10:30:00+00:00",
|
||||
"datetime": "2024-05-31T10:30:00+00:00",
|
||||
"uuid": "uuid-001",
|
||||
"event_properties": {},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m1"}}, "attributions": {"data": []}},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/events", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "event_001"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
latest_state = output.most_recent_state.stream_state.__dict__
|
||||
assert "datetime" in latest_state
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_with_prior_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test incremental sync with a prior state from previous sync.
|
||||
|
||||
Given: A previous sync state with a datetime cursor value
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use the state cursor and return only new/updated records
|
||||
"""
|
||||
# Using early start_date (before test data) so state cursor is used for filtering
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 1, 1, tzinfo=timezone.utc)).build()
|
||||
# State date within 7 days of _NOW (2024-06-01) to ensure only one stream slice is created
|
||||
# (events stream uses step: P7D windowing)
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"datetime": "2024-05-31T00:00:00+0000"}).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"fields[event]": "event_properties,timestamp,uuid,datetime",
|
||||
"fields[metric]": "name,created,updated,integration",
|
||||
"include": "metric,attributions",
|
||||
"filter": "greater-or-equal(datetime,2024-05-31T00:00:00+0000),less-or-equal(datetime,2024-06-01T12:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_new",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T10:00:00+00:00",
|
||||
"datetime": "2024-05-31T10:00:00+00:00",
|
||||
"uuid": "uuid-new",
|
||||
"event_properties": {},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m1"}}, "attributions": {"data": []}},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/events", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "event_new"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
|
||||
@HttpMocker()
|
||||
def test_transformation_adds_datetime_field(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the AddFields transformation correctly extracts 'datetime' from attributes.
|
||||
|
||||
The manifest configures:
|
||||
transformations:
|
||||
- type: AddFields
|
||||
fields:
|
||||
- path: [datetime]
|
||||
value: "{{ record.get('attributes', {}).get('datetime') }}"
|
||||
|
||||
Given: An event record with datetime in attributes
|
||||
When: Running a sync
|
||||
Then: The 'datetime' field should be added at the root level of the record
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"fields[event]": "event_properties,timestamp,uuid,datetime",
|
||||
"fields[metric]": "name,created,updated,integration",
|
||||
"include": "metric,attributions",
|
||||
"filter": "greater-or-equal(datetime,2024-05-31T00:00:00+0000),less-or-equal(datetime,2024-06-01T12:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_transform_test",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T14:45:00+00:00",
|
||||
"datetime": "2024-05-31T14:45:00+00:00",
|
||||
"uuid": "uuid-transform",
|
||||
"event_properties": {"test": "value"},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m1"}}, "attributions": {"data": []}},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/events", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert "datetime" in record
|
||||
assert record["datetime"] == "2024-05-31T14:45:00+00:00"
|
||||
assert record["attributes"]["datetime"] == "2024-05-31T14:45:00+00:00"
|
||||
|
||||
@HttpMocker()
|
||||
def test_rate_limit_429_handling(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles 429 rate limit responses with RATE_LIMITED action.
|
||||
|
||||
Given: An API that returns a 429 rate limit error
|
||||
When: Making an API request
|
||||
Then: The connector should respect the Retry-After header and retry
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"fields[event]": "event_properties,timestamp,uuid,datetime",
|
||||
"fields[metric]": "name,created,updated,integration",
|
||||
"include": "metric,attributions",
|
||||
"filter": "greater-or-equal(datetime,2024-05-31T00:00:00+0000),less-or-equal(datetime,2024-06-01T12:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
[
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Rate limit exceeded"}]}),
|
||||
status_code=429,
|
||||
headers={"Retry-After": "1"},
|
||||
),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_after_retry",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T10:00:00+00:00",
|
||||
"datetime": "2024-05-31T10:00:00+00:00",
|
||||
"uuid": "uuid-retry",
|
||||
"event_properties": {},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m1"}}, "attributions": {"data": []}},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/events", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "event_after_retry"
|
||||
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
# Check for backoff log message pattern
|
||||
assert any(
|
||||
"Backing off" in msg and "UserDefinedBackoffException" in msg and "429" in msg for msg in log_messages
|
||||
), "Expected backoff log message for 429 rate limit"
|
||||
# Check for retry/sleeping log message pattern
|
||||
assert any(
|
||||
"Sleeping for" in msg and "seconds" in msg for msg in log_messages
|
||||
), "Expected retry sleeping log message for 429 rate limit"
|
||||
|
||||
@HttpMocker()
|
||||
def test_unauthorized_401_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 401 Unauthorized errors with FAIL action.
|
||||
|
||||
Given: Invalid API credentials
|
||||
When: Making an API request that returns 401
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key("invalid_key").with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint("invalid_key")
|
||||
.with_query_params(
|
||||
{
|
||||
"fields[event]": "event_properties,timestamp,uuid,datetime",
|
||||
"fields[metric]": "name,created,updated,integration",
|
||||
"include": "metric,attributions",
|
||||
"filter": "greater-or-equal(datetime,2024-05-31T00:00:00+0000),less-or-equal(datetime,2024-06-01T12:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Invalid API key"}]}),
|
||||
status_code=401,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 401 authentication failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_forbidden_403_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 403 Forbidden errors with FAIL action.
|
||||
|
||||
The manifest configures 403 errors with action: FAIL, which means the connector
|
||||
should fail the sync when permission errors occur.
|
||||
|
||||
Given: API credentials with insufficient permissions
|
||||
When: Making an API request that returns 403
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"fields[event]": "event_properties,timestamp,uuid,datetime",
|
||||
"fields[metric]": "name,created,updated,integration",
|
||||
"include": "metric,attributions",
|
||||
"filter": "greater-or-equal(datetime,2024-05-31T00:00:00+0000),less-or-equal(datetime,2024-06-01T12:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Forbidden - insufficient permissions"}]}),
|
||||
status_code=403,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 403 permission failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_empty_results(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles empty results gracefully.
|
||||
|
||||
Given: An API that returns no events
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should return zero records without errors
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"fields[event]": "event_properties,timestamp,uuid,datetime",
|
||||
"fields[metric]": "name,created,updated,integration",
|
||||
"include": "metric,attributions",
|
||||
"filter": "greater-or-equal(datetime,2024-05-31T00:00:00+0000),less-or-equal(datetime,2024-06-01T12:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"data": [], "links": {"self": "https://a.klaviyo.com/api/events", "next": None}}),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 0
|
||||
assert not any(log.log.level == "ERROR" for log in output.logs)
|
||||
@@ -0,0 +1,545 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest import TestCase
|
||||
|
||||
import freezegun
|
||||
from unit_tests.conftest import get_source
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from airbyte_cdk.test.catalog_builder import CatalogBuilder
|
||||
from airbyte_cdk.test.entrypoint_wrapper import read
|
||||
from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse
|
||||
from airbyte_cdk.test.state_builder import StateBuilder
|
||||
from integration.config import ConfigBuilder
|
||||
from integration.request_builder import KlaviyoRequestBuilder
|
||||
from integration.response_builder import KlaviyoPaginatedResponseBuilder
|
||||
|
||||
|
||||
_NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
_STREAM_NAME = "events_detailed"
|
||||
_API_KEY = "test_api_key_abc123"
|
||||
|
||||
|
||||
@freezegun.freeze_time(_NOW.isoformat())
|
||||
class TestEventsDetailedStream(TestCase):
|
||||
"""
|
||||
Tests for the Klaviyo 'events_detailed' stream.
|
||||
|
||||
Stream configuration from manifest.yaml:
|
||||
- Uses CustomRecordExtractor to flatten included metric data into event records
|
||||
- Incremental sync with DatetimeBasedCursor on 'datetime' field
|
||||
- Request parameters: fields[event], fields[metric], include=metric
|
||||
- Pagination: CursorPagination
|
||||
- Error handling: 429 RATE_LIMITED, 401/403 FAIL
|
||||
- Transformations: AddFields to extract 'datetime' from attributes
|
||||
"""
|
||||
|
||||
@HttpMocker()
|
||||
def test_full_refresh_with_included_metrics(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test full refresh sync with included metric data.
|
||||
|
||||
The CustomRecordExtractor flattens the included metric data into each event record.
|
||||
|
||||
Given: An API response with events and included metrics
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should return events with metric data merged in
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# events_detailed stream uses include, fields[metric], filter, and sort query parameters
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"include": "metric,attributions",
|
||||
"fields[metric]": "name",
|
||||
"filter": "greater-than(datetime,2024-05-31T00:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_001",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T10:30:00+00:00",
|
||||
"datetime": "2024-05-31T10:30:00+00:00",
|
||||
"uuid": "550e8400-e29b-41d4-a716-446655440000",
|
||||
"event_properties": {"value": 99.99, "currency": "USD"},
|
||||
},
|
||||
"relationships": {
|
||||
"metric": {"data": {"type": "metric", "id": "metric_001"}},
|
||||
"attributions": {"data": []},
|
||||
},
|
||||
}
|
||||
],
|
||||
"included": [
|
||||
{
|
||||
"type": "metric",
|
||||
"id": "metric_001",
|
||||
"attributes": {
|
||||
"name": "Placed Order",
|
||||
"created": "2023-01-01T00:00:00+00:00",
|
||||
"updated": "2024-01-01T00:00:00+00:00",
|
||||
"integration": {"id": "integration_001", "name": "Shopify"},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/events", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert record["id"] == "event_001"
|
||||
assert record["attributes"]["uuid"] == "550e8400-e29b-41d4-a716-446655440000"
|
||||
|
||||
@HttpMocker()
|
||||
def test_pagination_multiple_pages(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fetches all pages when pagination is present.
|
||||
|
||||
Given: An API that returns multiple pages of events with included metrics
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should follow pagination links and return all records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Use a single mock with multiple responses to avoid ambiguity in mock matching.
|
||||
# The first response includes a next link, the second response has no next link.
|
||||
# events_detailed stream uses include, fields[metric], filter, and sort query parameters
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
[
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_001",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T10:00:00+00:00",
|
||||
"datetime": "2024-05-31T10:00:00+00:00",
|
||||
"uuid": "uuid-001",
|
||||
"event_properties": {},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m1"}}, "attributions": {"data": []}},
|
||||
}
|
||||
],
|
||||
"included": [{"type": "metric", "id": "m1", "attributes": {"name": "Metric 1"}}],
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/events",
|
||||
"next": "https://a.klaviyo.com/api/events?page[cursor]=abc123",
|
||||
},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_002",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T11:00:00+00:00",
|
||||
"datetime": "2024-05-31T11:00:00+00:00",
|
||||
"uuid": "uuid-002",
|
||||
"event_properties": {},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m2"}}, "attributions": {"data": []}},
|
||||
}
|
||||
],
|
||||
"included": [{"type": "metric", "id": "m2", "attributes": {"name": "Metric 2"}}],
|
||||
"links": {"self": "https://a.klaviyo.com/api/events?page[cursor]=abc123", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 2
|
||||
assert output.records[0].record.data["id"] == "event_001"
|
||||
assert output.records[1].record.data["id"] == "event_002"
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_first_sync_no_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test first incremental sync with no previous state.
|
||||
|
||||
Given: No previous state (first sync)
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use start_date from config and emit state message
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# events_detailed stream uses include, fields[metric], filter, and sort query parameters
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"include": "metric,attributions",
|
||||
"fields[metric]": "name",
|
||||
"filter": "greater-than(datetime,2024-05-31T00:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_001",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T10:30:00+00:00",
|
||||
"datetime": "2024-05-31T10:30:00+00:00",
|
||||
"uuid": "uuid-001",
|
||||
"event_properties": {},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m1"}}, "attributions": {"data": []}},
|
||||
}
|
||||
],
|
||||
"included": [{"type": "metric", "id": "m1", "attributes": {"name": "Metric 1"}}],
|
||||
"links": {"self": "https://a.klaviyo.com/api/events", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "event_001"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
latest_state = output.most_recent_state.stream_state.__dict__
|
||||
assert "datetime" in latest_state
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_with_prior_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test incremental sync with a prior state from previous sync.
|
||||
|
||||
Given: A previous sync state with a datetime cursor value
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use the state cursor and return only new/updated records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"datetime": "2024-03-01T00:00:00+00:00"}).build()
|
||||
|
||||
# events_detailed stream uses include, fields[metric], filter, and sort query parameters
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"include": "metric,attributions",
|
||||
"fields[metric]": "name",
|
||||
"filter": "greater-than(datetime,2024-05-31T00:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_new",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T10:00:00+00:00",
|
||||
"datetime": "2024-05-31T10:00:00+00:00",
|
||||
"uuid": "uuid-new",
|
||||
"event_properties": {},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m1"}}, "attributions": {"data": []}},
|
||||
}
|
||||
],
|
||||
"included": [{"type": "metric", "id": "m1", "attributes": {"name": "Metric 1"}}],
|
||||
"links": {"self": "https://a.klaviyo.com/api/events", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "event_new"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
|
||||
@HttpMocker()
|
||||
def test_transformation_adds_datetime_field(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the AddFields transformation correctly extracts 'datetime' from attributes.
|
||||
|
||||
Given: An event record with datetime in attributes
|
||||
When: Running a sync
|
||||
Then: The 'datetime' field should be added at the root level of the record
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# events_detailed stream uses include, fields[metric], filter, and sort query parameters
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"include": "metric,attributions",
|
||||
"fields[metric]": "name",
|
||||
"filter": "greater-than(datetime,2024-05-31T00:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_transform_test",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T14:45:00+00:00",
|
||||
"datetime": "2024-05-31T14:45:00+00:00",
|
||||
"uuid": "uuid-transform",
|
||||
"event_properties": {"test": "value"},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m1"}}, "attributions": {"data": []}},
|
||||
}
|
||||
],
|
||||
"included": [{"type": "metric", "id": "m1", "attributes": {"name": "Metric 1"}}],
|
||||
"links": {"self": "https://a.klaviyo.com/api/events", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert "datetime" in record
|
||||
assert record["datetime"] == "2024-05-31T14:45:00+00:00"
|
||||
|
||||
@HttpMocker()
|
||||
def test_rate_limit_429_handling(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles 429 rate limit responses with RATE_LIMITED action.
|
||||
|
||||
Given: An API that returns a 429 rate limit error
|
||||
When: Making an API request
|
||||
Then: The connector should respect the Retry-After header and retry
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# events_detailed stream uses include, fields[metric], filter, and sort query parameters
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"include": "metric,attributions",
|
||||
"fields[metric]": "name",
|
||||
"filter": "greater-than(datetime,2024-05-31T00:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
[
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Rate limit exceeded"}]}),
|
||||
status_code=429,
|
||||
headers={"Retry-After": "1"},
|
||||
),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_after_retry",
|
||||
"attributes": {
|
||||
"timestamp": "2024-05-31T10:00:00+00:00",
|
||||
"datetime": "2024-05-31T10:00:00+00:00",
|
||||
"uuid": "uuid-retry",
|
||||
"event_properties": {},
|
||||
},
|
||||
"relationships": {"metric": {"data": {"type": "metric", "id": "m1"}}, "attributions": {"data": []}},
|
||||
}
|
||||
],
|
||||
"included": [{"type": "metric", "id": "m1", "attributes": {"name": "Metric 1"}}],
|
||||
"links": {"self": "https://a.klaviyo.com/api/events", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "event_after_retry"
|
||||
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
# Check for backoff log message pattern
|
||||
assert any(
|
||||
"Backing off" in msg and "UserDefinedBackoffException" in msg and "429" in msg for msg in log_messages
|
||||
), "Expected backoff log message for 429 rate limit"
|
||||
# Check for retry/sleeping log message pattern
|
||||
assert any(
|
||||
"Sleeping for" in msg and "seconds" in msg for msg in log_messages
|
||||
), "Expected retry sleeping log message for 429 rate limit"
|
||||
|
||||
@HttpMocker()
|
||||
def test_unauthorized_401_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 401 Unauthorized errors with FAIL action.
|
||||
|
||||
Given: Invalid API credentials
|
||||
When: Making an API request that returns 401
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key("invalid_key").with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# events_detailed stream uses include, fields[metric], filter, and sort query parameters
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint("invalid_key")
|
||||
.with_query_params(
|
||||
{
|
||||
"include": "metric,attributions",
|
||||
"fields[metric]": "name",
|
||||
"filter": "greater-than(datetime,2024-05-31T00:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Invalid API key"}]}),
|
||||
status_code=401,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 401 authentication failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_forbidden_403_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 403 Forbidden errors with FAIL action.
|
||||
|
||||
The manifest configures 403 errors with action: FAIL, which means the connector
|
||||
should fail the sync when permission errors occur.
|
||||
|
||||
Given: API credentials with insufficient permissions
|
||||
When: Making an API request that returns 403
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# events_detailed stream uses include, fields[metric], filter, and sort query parameters
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"include": "metric,attributions",
|
||||
"fields[metric]": "name",
|
||||
"filter": "greater-than(datetime,2024-05-31T00:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Forbidden - insufficient permissions"}]}),
|
||||
status_code=403,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 403 permission failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_empty_results(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles empty results gracefully.
|
||||
|
||||
Given: An API that returns no events
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should return zero records without errors
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# events_detailed stream uses include, fields[metric], filter, and sort query parameters
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.events_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"include": "metric,attributions",
|
||||
"fields[metric]": "name",
|
||||
"filter": "greater-than(datetime,2024-05-31T00:00:00+0000)",
|
||||
"sort": "datetime",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"data": [], "included": [], "links": {"self": "https://a.klaviyo.com/api/events", "next": None}}),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 0
|
||||
assert not any(log.log.level == "ERROR" for log in output.logs)
|
||||
@@ -0,0 +1,640 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest import TestCase
|
||||
|
||||
import freezegun
|
||||
from unit_tests.conftest import get_source
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from airbyte_cdk.test.catalog_builder import CatalogBuilder
|
||||
from airbyte_cdk.test.entrypoint_wrapper import read
|
||||
from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse
|
||||
from airbyte_cdk.test.state_builder import StateBuilder
|
||||
from integration.config import ConfigBuilder
|
||||
from integration.request_builder import KlaviyoRequestBuilder
|
||||
from integration.response_builder import KlaviyoPaginatedResponseBuilder
|
||||
|
||||
|
||||
_NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
_STREAM_NAME = "flows"
|
||||
_API_KEY = "test_api_key_abc123"
|
||||
|
||||
|
||||
@freezegun.freeze_time(_NOW.isoformat())
|
||||
class TestFlowsStream(TestCase):
|
||||
"""
|
||||
Tests for the Klaviyo 'flows' stream.
|
||||
|
||||
Stream configuration from manifest.yaml:
|
||||
- Uses ListPartitionRouter to iterate over flow statuses (draft, manual, live)
|
||||
- Incremental sync with DatetimeBasedCursor on 'updated' field
|
||||
- Pagination: CursorPagination
|
||||
- Error handling: 429 RATE_LIMITED, 401/403 FAIL
|
||||
- Transformations: AddFields to extract 'updated' from attributes
|
||||
"""
|
||||
|
||||
@HttpMocker()
|
||||
def test_full_refresh_single_page(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test full refresh sync with a single page of results.
|
||||
|
||||
Given: A configured Klaviyo connector
|
||||
When: Running a full refresh sync for the flows stream
|
||||
Then: The connector should make requests for each flow status partition
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock both partitions (archived: true/false)
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.flows_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated,2024-05-31T00:00:00+0000),less-or-equal(updated,2024-06-01T12:00:00+0000),equals(archived,{archived}))",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "flow",
|
||||
"id": "flow_001",
|
||||
"attributes": {
|
||||
"name": "Welcome Series",
|
||||
"status": "live",
|
||||
"archived": False,
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"trigger_type": "List",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/flows", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 2
|
||||
record = output.records[0].record.data
|
||||
assert record["id"] == "flow_001"
|
||||
assert record["attributes"]["name"] == "Welcome Series"
|
||||
|
||||
@HttpMocker()
|
||||
def test_partition_router_multiple_statuses(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the ListPartitionRouter correctly iterates over all flow statuses.
|
||||
|
||||
The manifest configures:
|
||||
partition_router:
|
||||
type: ListPartitionRouter
|
||||
values: ["draft", "manual", "live"]
|
||||
cursor_field: "status"
|
||||
|
||||
Given: An API that returns flows for each status
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should make requests for each status partition
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock both partitions (archived: true/false)
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.flows_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated,2024-05-31T00:00:00+0000),less-or-equal(updated,2024-06-01T12:00:00+0000),equals(archived,{archived}))",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "flow",
|
||||
"id": "flow_001",
|
||||
"attributes": {
|
||||
"name": "Test Flow",
|
||||
"status": "live",
|
||||
"archived": False,
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"trigger_type": "Segment",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/flows", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 2
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "flow_001" in record_ids
|
||||
|
||||
@HttpMocker()
|
||||
def test_pagination_multiple_pages(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fetches all pages when pagination is present.
|
||||
|
||||
Given: An API that returns multiple pages of flows
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should follow pagination links and return all records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Use with_any_query_params() because pagination requests add page[cursor] param
|
||||
# which differs from the initial request's filter/sort params.
|
||||
# The flows stream has 2 partitions (archived=true/false) and each partition has 2 pages,
|
||||
# so we need 4 responses total.
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.flows_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
[
|
||||
# Partition 1 (archived=true), Page 1
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "flow",
|
||||
"id": "flow_001",
|
||||
"attributes": {
|
||||
"name": "Flow 1",
|
||||
"status": "live",
|
||||
"archived": True,
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
"trigger_type": "List",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.with_next_page_link("https://a.klaviyo.com/api/flows?page[cursor]=abc123")
|
||||
.build(),
|
||||
# Partition 1 (archived=true), Page 2
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "flow",
|
||||
"id": "flow_002",
|
||||
"attributes": {
|
||||
"name": "Flow 2",
|
||||
"status": "live",
|
||||
"archived": True,
|
||||
"created": "2024-05-31T11:00:00+00:00",
|
||||
"updated": "2024-05-31T11:00:00+00:00",
|
||||
"trigger_type": "Segment",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.build(),
|
||||
# Partition 2 (archived=false), Page 1
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "flow",
|
||||
"id": "flow_003",
|
||||
"attributes": {
|
||||
"name": "Flow 3",
|
||||
"status": "live",
|
||||
"archived": False,
|
||||
"created": "2024-05-31T12:00:00+00:00",
|
||||
"updated": "2024-05-31T12:00:00+00:00",
|
||||
"trigger_type": "List",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.with_next_page_link("https://a.klaviyo.com/api/flows?page[cursor]=def456")
|
||||
.build(),
|
||||
# Partition 2 (archived=false), Page 2
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "flow",
|
||||
"id": "flow_004",
|
||||
"attributes": {
|
||||
"name": "Flow 4",
|
||||
"status": "live",
|
||||
"archived": False,
|
||||
"created": "2024-05-31T13:00:00+00:00",
|
||||
"updated": "2024-05-31T13:00:00+00:00",
|
||||
"trigger_type": "Segment",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.build(),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 4
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "flow_001" in record_ids
|
||||
assert "flow_002" in record_ids
|
||||
assert "flow_003" in record_ids
|
||||
assert "flow_004" in record_ids
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_first_sync_no_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test first incremental sync with no previous state.
|
||||
|
||||
Given: No previous state (first sync)
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use start_date from config and emit state message
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock both partitions (archived: true/false)
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.flows_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated,2024-05-31T00:00:00+0000),less-or-equal(updated,2024-06-01T12:00:00+0000),equals(archived,{archived}))",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "flow",
|
||||
"id": "flow_001",
|
||||
"attributes": {
|
||||
"name": "Test Flow",
|
||||
"status": "live",
|
||||
"archived": False,
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"trigger_type": "List",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/flows", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 2
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "flow_001" in record_ids
|
||||
assert len(output.state_messages) > 0
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_with_prior_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test incremental sync with a prior state from previous sync.
|
||||
|
||||
Given: A previous sync state with an updated cursor value
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use the state cursor and return only new/updated records
|
||||
"""
|
||||
# Use start_date very close to _NOW to ensure only 1 time slice (flows uses step: P30D)
|
||||
# With start_date=2024-05-25 and _NOW=2024-06-01, we get <30 days = 1 time slice
|
||||
# Combined with 2 partitions (archived=true/false), this creates exactly 2 requests
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 25, tzinfo=timezone.utc)).build()
|
||||
# State date within the time window
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": "2024-05-31T00:00:00+0000"}).build()
|
||||
|
||||
# Use with_any_query_params() because the exact filter string depends on state cursor
|
||||
# and time windowing logic. The flows stream has 2 partitions (archived=true/false).
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.flows_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
[
|
||||
# Partition 1 (archived=true)
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "flow",
|
||||
"id": "flow_new_1",
|
||||
"attributes": {
|
||||
"name": "New Flow 1",
|
||||
"status": "live",
|
||||
"archived": True,
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
"trigger_type": "Segment",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/flows", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
# Partition 2 (archived=false)
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "flow",
|
||||
"id": "flow_new_2",
|
||||
"attributes": {
|
||||
"name": "New Flow 2",
|
||||
"status": "live",
|
||||
"archived": False,
|
||||
"created": "2024-05-31T11:00:00+00:00",
|
||||
"updated": "2024-05-31T11:00:00+00:00",
|
||||
"trigger_type": "List",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/flows", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
assert len(output.records) == 2
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "flow_new_1" in record_ids
|
||||
assert "flow_new_2" in record_ids
|
||||
assert len(output.state_messages) > 0
|
||||
|
||||
@HttpMocker()
|
||||
def test_transformation_adds_updated_field(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the AddFields transformation correctly extracts 'updated' from attributes.
|
||||
|
||||
Given: A flow record with updated in attributes
|
||||
When: Running a sync
|
||||
Then: The 'updated' field should be added at the root level of the record
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock both partitions (archived: true/false)
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.flows_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated,2024-05-31T00:00:00+0000),less-or-equal(updated,2024-06-01T12:00:00+0000),equals(archived,{archived}))",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "flow",
|
||||
"id": "flow_transform_test",
|
||||
"attributes": {
|
||||
"name": "Transform Test",
|
||||
"status": "live",
|
||||
"archived": False,
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T14:45:00+00:00",
|
||||
"trigger_type": "List",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/flows", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 2
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "flow_transform_test" in record_ids
|
||||
record = output.records[0].record.data
|
||||
assert "updated" in record
|
||||
assert record["updated"] == "2024-05-31T14:45:00+00:00"
|
||||
|
||||
@HttpMocker()
|
||||
def test_rate_limit_429_handling(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles 429 rate limit responses with RATE_LIMITED action.
|
||||
|
||||
Given: An API that returns a 429 rate limit error
|
||||
When: Making an API request
|
||||
Then: The connector should respect the Retry-After header and retry
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock both partitions with rate limit handling
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.flows_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated,2024-05-31T00:00:00+0000),less-or-equal(updated,2024-06-01T12:00:00+0000),equals(archived,{archived}))",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
[
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Rate limit exceeded"}]}),
|
||||
status_code=429,
|
||||
headers={"Retry-After": "1"},
|
||||
),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "flow",
|
||||
"id": "flow_after_retry",
|
||||
"attributes": {
|
||||
"name": "After Retry",
|
||||
"status": "live",
|
||||
"archived": False,
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
"trigger_type": "List",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/flows", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 2
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "flow_after_retry" in record_ids
|
||||
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
# Check for backoff log message pattern
|
||||
assert any(
|
||||
"Backing off" in msg and "UserDefinedBackoffException" in msg and "429" in msg for msg in log_messages
|
||||
), "Expected backoff log message for 429 rate limit"
|
||||
# Check for retry/sleeping log message pattern
|
||||
assert any(
|
||||
"Sleeping for" in msg and "seconds" in msg for msg in log_messages
|
||||
), "Expected retry sleeping log message for 429 rate limit"
|
||||
|
||||
@HttpMocker()
|
||||
def test_unauthorized_401_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 401 Unauthorized errors with FAIL action.
|
||||
|
||||
Given: Invalid API credentials
|
||||
When: Making an API request that returns 401
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key("invalid_key").with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock both partitions with 401 error
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.flows_endpoint("invalid_key")
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated,2024-05-31T00:00:00+0000),less-or-equal(updated,2024-06-01T12:00:00+0000),equals(archived,{archived}))",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Invalid API key"}]}),
|
||||
status_code=401,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 401 authentication failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_forbidden_403_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 403 Forbidden errors with FAIL action.
|
||||
|
||||
The manifest configures 403 errors with action: FAIL, which means the connector
|
||||
should fail the sync when permission errors occur.
|
||||
|
||||
Given: API credentials with insufficient permissions
|
||||
When: Making an API request that returns 403
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock both partitions with 403 error
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.flows_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated,2024-05-31T00:00:00+0000),less-or-equal(updated,2024-06-01T12:00:00+0000),equals(archived,{archived}))",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Forbidden - insufficient permissions"}]}),
|
||||
status_code=403,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 403 permission failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_empty_results(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles empty results gracefully.
|
||||
|
||||
Given: An API that returns no flows
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should return zero records without errors
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Mock both partitions with empty results
|
||||
for archived in ["true", "false"]:
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.flows_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": f"and(greater-or-equal(updated,2024-05-31T00:00:00+0000),less-or-equal(updated,2024-06-01T12:00:00+0000),equals(archived,{archived}))",
|
||||
"sort": "updated",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"data": [], "links": {"self": "https://a.klaviyo.com/api/flows", "next": None}}),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 0
|
||||
assert not any(log.log.level == "ERROR" for log in output.logs)
|
||||
@@ -0,0 +1,551 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest import TestCase
|
||||
|
||||
import freezegun
|
||||
from unit_tests.conftest import get_source
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from airbyte_cdk.test.catalog_builder import CatalogBuilder
|
||||
from airbyte_cdk.test.entrypoint_wrapper import read
|
||||
from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse
|
||||
from airbyte_cdk.test.state_builder import StateBuilder
|
||||
from integration.config import ConfigBuilder
|
||||
from integration.request_builder import KlaviyoRequestBuilder
|
||||
from integration.response_builder import KlaviyoPaginatedResponseBuilder
|
||||
|
||||
|
||||
_NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
_STREAM_NAME = "global_exclusions"
|
||||
_API_KEY = "test_api_key_abc123"
|
||||
|
||||
|
||||
@freezegun.freeze_time(_NOW.isoformat())
|
||||
class TestGlobalExclusionsStream(TestCase):
|
||||
"""
|
||||
Tests for the Klaviyo 'global_exclusions' stream.
|
||||
|
||||
Stream configuration from manifest.yaml:
|
||||
- Uses /profiles endpoint with additional-fields[profile]: subscriptions
|
||||
- RecordFilter: Only returns profiles with suppression data
|
||||
- Transformations:
|
||||
- AddFields: extracts 'updated' from attributes
|
||||
- AddFields: copies suppression to suppressions (plural)
|
||||
- RemoveFields: removes original suppression field
|
||||
- Error handling: 429 RATE_LIMITED, 401/403 FAIL
|
||||
- Pagination: CursorPagination
|
||||
"""
|
||||
|
||||
@HttpMocker()
|
||||
def test_full_refresh_filters_suppressed_profiles(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that record_filter correctly filters only suppressed profiles.
|
||||
|
||||
The manifest configures:
|
||||
record_filter:
|
||||
type: RecordFilter
|
||||
condition: "{{ record['attributes']['subscriptions']['email']['marketing']['suppression'] }}"
|
||||
|
||||
Given: API returns profiles with and without suppression
|
||||
When: Running a full refresh sync
|
||||
Then: Only profiles with suppression data should be returned
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Global exclusions stream uses profiles endpoint with additional-fields[profile]: subscriptions
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params({"additional-fields[profile]": "subscriptions", "page[size]": "100"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_suppressed",
|
||||
"attributes": {
|
||||
"email": "suppressed@example.com",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"subscriptions": {
|
||||
"email": {
|
||||
"marketing": {
|
||||
"can_receive_email_marketing": False,
|
||||
"consent": "UNSUBSCRIBED",
|
||||
"suppression": [{"reason": "USER_SUPPRESSED", "timestamp": "2024-05-31T10:00:00+00:00"}],
|
||||
}
|
||||
},
|
||||
"sms": {"marketing": {"can_receive_sms_marketing": False}},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_not_suppressed",
|
||||
"attributes": {
|
||||
"email": "active@example.com",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"subscriptions": {
|
||||
"email": {
|
||||
"marketing": {
|
||||
"can_receive_email_marketing": True,
|
||||
"consent": "SUBSCRIBED",
|
||||
"suppression": [],
|
||||
}
|
||||
},
|
||||
"sms": {"marketing": {"can_receive_sms_marketing": True}},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert record["id"] == "profile_suppressed"
|
||||
assert record["attributes"]["email"] == "suppressed@example.com"
|
||||
|
||||
@HttpMocker()
|
||||
def test_transformation_adds_suppressions_field(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that transformations correctly add 'suppressions' and remove 'suppression'.
|
||||
|
||||
The manifest configures:
|
||||
transformations:
|
||||
- type: AddFields (copies suppression to suppressions)
|
||||
- type: RemoveFields (removes original suppression)
|
||||
|
||||
Given: A suppressed profile record
|
||||
When: Running a sync
|
||||
Then: The record should have 'suppressions' field and no 'suppression' field
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Global exclusions stream uses profiles endpoint with additional-fields[profile]: subscriptions
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params({"additional-fields[profile]": "subscriptions", "page[size]": "100"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_transform_test",
|
||||
"attributes": {
|
||||
"email": "transform@example.com",
|
||||
"updated": "2024-05-31T14:45:00+00:00",
|
||||
"subscriptions": {
|
||||
"email": {
|
||||
"marketing": {
|
||||
"can_receive_email_marketing": False,
|
||||
"consent": "UNSUBSCRIBED",
|
||||
"suppression": [{"reason": "HARD_BOUNCE", "timestamp": "2024-05-31T10:00:00+00:00"}],
|
||||
}
|
||||
},
|
||||
"sms": {"marketing": {"can_receive_sms_marketing": False}},
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
|
||||
assert "updated" in record
|
||||
assert record["updated"] == "2024-05-31T14:45:00+00:00"
|
||||
|
||||
marketing = record["attributes"]["subscriptions"]["email"]["marketing"]
|
||||
assert "suppressions" in marketing
|
||||
assert len(marketing["suppressions"]) == 1
|
||||
assert marketing["suppressions"][0]["reason"] == "HARD_BOUNCE"
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_first_sync_no_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test first incremental sync with no previous state.
|
||||
|
||||
Given: No previous state (first sync)
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use start_date from config and emit state message
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Global exclusions stream uses profiles endpoint with additional-fields[profile]: subscriptions
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params({"additional-fields[profile]": "subscriptions", "page[size]": "100"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_001",
|
||||
"attributes": {
|
||||
"email": "test@example.com",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"subscriptions": {
|
||||
"email": {
|
||||
"marketing": {
|
||||
"suppression": [{"reason": "USER_SUPPRESSED", "timestamp": "2024-05-31T10:00:00+00:00"}]
|
||||
}
|
||||
},
|
||||
"sms": {"marketing": {}},
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert len(output.state_messages) > 0
|
||||
latest_state = output.most_recent_state.stream_state.__dict__
|
||||
assert "updated" in latest_state
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_with_prior_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test incremental sync with a prior state from previous sync.
|
||||
|
||||
Given: A previous sync state with an updated cursor value
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use the state cursor and return only new/updated records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": "2024-05-30T00:00:00+00:00"}).build()
|
||||
|
||||
# Global exclusions stream uses profiles endpoint with additional-fields[profile]: subscriptions
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params({"additional-fields[profile]": "subscriptions", "page[size]": "100"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_new",
|
||||
"attributes": {
|
||||
"email": "new@example.com",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
"subscriptions": {
|
||||
"email": {
|
||||
"marketing": {
|
||||
"suppression": [{"reason": "SPAM_COMPLAINT", "timestamp": "2024-05-31T09:00:00+00:00"}]
|
||||
}
|
||||
},
|
||||
"sms": {"marketing": {}},
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "profile_new"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
latest_state = output.most_recent_state.stream_state.__dict__
|
||||
# Note: The connector returns datetime with +0000 format (without colon)
|
||||
assert latest_state["updated"] == "2024-05-31T10:00:00+0000"
|
||||
|
||||
@HttpMocker()
|
||||
def test_pagination_multiple_pages(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fetches all pages when pagination is present.
|
||||
|
||||
Given: An API that returns multiple pages of suppressed profiles
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should follow pagination links and return all records
|
||||
|
||||
Note: Uses with_any_query_params() because pagination adds page[cursor] to the
|
||||
request params, making exact matching impractical.
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Use with_any_query_params() since pagination adds page[cursor] dynamically
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
[
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_001",
|
||||
"attributes": {
|
||||
"email": "user1@example.com",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
"subscriptions": {
|
||||
"email": {
|
||||
"marketing": {
|
||||
"suppression": [{"reason": "USER_SUPPRESSED", "timestamp": "2024-05-31T09:00:00+00:00"}]
|
||||
}
|
||||
},
|
||||
"sms": {"marketing": {}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.with_next_page_link("https://a.klaviyo.com/api/profiles?page[cursor]=abc123")
|
||||
.build(),
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_002",
|
||||
"attributes": {
|
||||
"email": "user2@example.com",
|
||||
"updated": "2024-05-31T11:00:00+00:00",
|
||||
"subscriptions": {
|
||||
"email": {
|
||||
"marketing": {"suppression": [{"reason": "HARD_BOUNCE", "timestamp": "2024-05-31T10:00:00+00:00"}]}
|
||||
},
|
||||
"sms": {"marketing": {}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.build(),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 2
|
||||
assert output.records[0].record.data["id"] == "profile_001"
|
||||
assert output.records[1].record.data["id"] == "profile_002"
|
||||
|
||||
@HttpMocker()
|
||||
def test_rate_limit_429_handling(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles 429 rate limit responses with RATE_LIMITED action.
|
||||
|
||||
Given: An API that returns a 429 rate limit error
|
||||
When: Making an API request
|
||||
Then: The connector should respect the Retry-After header and retry
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Global exclusions stream uses profiles endpoint with additional-fields[profile]: subscriptions
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params({"additional-fields[profile]": "subscriptions", "page[size]": "100"})
|
||||
.build(),
|
||||
[
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Rate limit exceeded"}]}),
|
||||
status_code=429,
|
||||
headers={"Retry-After": "1"},
|
||||
),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_after_retry",
|
||||
"attributes": {
|
||||
"email": "retry@example.com",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
"subscriptions": {
|
||||
"email": {
|
||||
"marketing": {
|
||||
"suppression": [{"reason": "USER_SUPPRESSED", "timestamp": "2024-05-31T09:00:00+00:00"}]
|
||||
}
|
||||
},
|
||||
"sms": {"marketing": {}},
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "profile_after_retry"
|
||||
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
# Check for backoff log message pattern
|
||||
assert any(
|
||||
"Backing off" in msg and "UserDefinedBackoffException" in msg and "429" in msg for msg in log_messages
|
||||
), "Expected backoff log message for 429 rate limit"
|
||||
# Check for retry/sleeping log message pattern
|
||||
assert any(
|
||||
"Sleeping for" in msg and "seconds" in msg for msg in log_messages
|
||||
), "Expected retry sleeping log message for 429 rate limit"
|
||||
|
||||
@HttpMocker()
|
||||
def test_unauthorized_401_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 401 Unauthorized errors with FAIL action.
|
||||
|
||||
Given: Invalid API credentials
|
||||
When: Making an API request that returns 401
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key("invalid_key").with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Global exclusions stream uses profiles endpoint with additional-fields[profile]: subscriptions
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint("invalid_key")
|
||||
.with_query_params({"additional-fields[profile]": "subscriptions", "page[size]": "100"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Invalid API key"}]}),
|
||||
status_code=401,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 401 authentication failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_forbidden_403_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 403 Forbidden errors with FAIL action.
|
||||
|
||||
The manifest configures 403 errors with action: FAIL, which means the connector
|
||||
should fail the sync when permission errors occur.
|
||||
|
||||
Given: API credentials with insufficient permissions
|
||||
When: Making an API request that returns 403
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Global exclusions stream uses profiles endpoint with additional-fields[profile]: subscriptions
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params({"additional-fields[profile]": "subscriptions", "page[size]": "100"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Forbidden - insufficient permissions"}]}),
|
||||
status_code=403,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 403 permission failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_empty_results_no_suppressed_profiles(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles empty results when no profiles are suppressed.
|
||||
|
||||
Given: An API that returns profiles but none are suppressed
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should return zero records without errors
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Global exclusions stream uses profiles endpoint with additional-fields[profile]: subscriptions
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params({"additional-fields[profile]": "subscriptions", "page[size]": "100"})
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_active",
|
||||
"attributes": {
|
||||
"email": "active@example.com",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"subscriptions": {
|
||||
"email": {
|
||||
"marketing": {"can_receive_email_marketing": True, "consent": "SUBSCRIBED", "suppression": []}
|
||||
},
|
||||
"sms": {"marketing": {"can_receive_sms_marketing": True}},
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 0
|
||||
assert not any(log.log.level == "ERROR" for log in output.logs)
|
||||
@@ -0,0 +1,498 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest import TestCase
|
||||
|
||||
import freezegun
|
||||
from unit_tests.conftest import get_source
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from airbyte_cdk.test.catalog_builder import CatalogBuilder
|
||||
from airbyte_cdk.test.entrypoint_wrapper import read
|
||||
from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse
|
||||
from airbyte_cdk.test.state_builder import StateBuilder
|
||||
from integration.config import ConfigBuilder
|
||||
from integration.request_builder import KlaviyoRequestBuilder
|
||||
from integration.response_builder import KlaviyoPaginatedResponseBuilder
|
||||
|
||||
|
||||
_NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
_STREAM_NAME = "lists"
|
||||
_API_KEY = "test_api_key_abc123"
|
||||
|
||||
|
||||
@freezegun.freeze_time(_NOW.isoformat())
|
||||
class TestListsStream(TestCase):
|
||||
"""
|
||||
Tests for the Klaviyo 'lists' stream.
|
||||
|
||||
Stream configuration from manifest.yaml:
|
||||
- Client-side incremental sync (is_client_side_incremental: true)
|
||||
- DatetimeBasedCursor on 'updated' field
|
||||
- is_data_feed: true - stops pagination when old records are detected
|
||||
- Pagination: CursorPagination
|
||||
- Error handling: 429 RATE_LIMITED, 401/403 FAIL
|
||||
- Transformations: AddFields to extract 'updated' from attributes
|
||||
"""
|
||||
|
||||
@HttpMocker()
|
||||
def test_full_refresh_single_page(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test full refresh sync with a single page of results.
|
||||
|
||||
Given: A configured Klaviyo connector
|
||||
When: Running a full refresh sync for the lists stream
|
||||
Then: The connector should make the correct API request and return all records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Lists stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_001",
|
||||
"attributes": {
|
||||
"name": "Newsletter Subscribers",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert record["id"] == "list_001"
|
||||
assert record["attributes"]["name"] == "Newsletter Subscribers"
|
||||
|
||||
@HttpMocker()
|
||||
def test_pagination_multiple_pages(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fetches all pages when pagination is present.
|
||||
|
||||
Given: An API that returns multiple pages of lists
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should follow pagination links and return all records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Use a single mock with multiple responses to avoid ambiguity in mock matching.
|
||||
# The first response includes a next_page_link, the second response has no next link.
|
||||
# Lists stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
[
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_001",
|
||||
"attributes": {
|
||||
"name": "List 1",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.with_next_page_link("https://a.klaviyo.com/api/lists?page[cursor]=abc123")
|
||||
.build(),
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_002",
|
||||
"attributes": {
|
||||
"name": "List 2",
|
||||
"created": "2024-05-31T11:00:00+00:00",
|
||||
"updated": "2024-05-31T11:00:00+00:00",
|
||||
"opt_in_process": "double_opt_in",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.build(),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 2
|
||||
assert output.records[0].record.data["id"] == "list_001"
|
||||
assert output.records[1].record.data["id"] == "list_002"
|
||||
|
||||
@HttpMocker()
|
||||
def test_client_side_incremental_first_sync_no_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test first incremental sync with no previous state (client-side incremental).
|
||||
|
||||
Given: No previous state (first sync)
|
||||
When: Running an incremental sync
|
||||
Then: The connector should fetch all records and emit state message
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Lists stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_001",
|
||||
"attributes": {
|
||||
"name": "Test List",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "list_001"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
latest_state = output.most_recent_state.stream_state.__dict__
|
||||
assert "updated" in latest_state
|
||||
|
||||
@HttpMocker()
|
||||
def test_client_side_incremental_with_prior_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test client-side incremental sync with a prior state from previous sync.
|
||||
|
||||
For client-side incremental streams (is_client_side_incremental: true), the connector
|
||||
fetches all records from the API but filters them client-side based on the state.
|
||||
|
||||
Given: A previous sync state with an updated cursor value
|
||||
When: Running an incremental sync
|
||||
Then: The connector should filter records client-side and only return new/updated records
|
||||
"""
|
||||
# Using early start_date (before test data) so state cursor is used for filtering
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 1, 1, tzinfo=timezone.utc)).build()
|
||||
# Using +0000 format (without colon) to match connector's timezone format
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": "2024-03-01T00:00:00+0000"}).build()
|
||||
|
||||
# Lists stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_old",
|
||||
"attributes": {
|
||||
"name": "Old List",
|
||||
"created": "2024-01-01T10:00:00+00:00",
|
||||
"updated": "2024-02-15T10:00:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_new",
|
||||
"attributes": {
|
||||
"name": "New List",
|
||||
"created": "2024-03-10T10:00:00+00:00",
|
||||
"updated": "2024-03-15T10:00:00+00:00",
|
||||
"opt_in_process": "double_opt_in",
|
||||
},
|
||||
},
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "list_new"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
latest_state = output.most_recent_state.stream_state.__dict__
|
||||
# Note: The connector returns datetime with +0000 format (without colon)
|
||||
assert latest_state["updated"] == "2024-03-15T10:00:00+0000"
|
||||
|
||||
@HttpMocker()
|
||||
def test_data_feed_stops_pagination_on_old_records(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that pagination stops when old records are detected (is_data_feed: true).
|
||||
|
||||
For data feed streams, if Page 1 contains records older than state, Page 2 should not be fetched.
|
||||
|
||||
Given: A state with a cursor value and API returning old records
|
||||
When: Running an incremental sync
|
||||
Then: The connector should stop pagination when old records are detected
|
||||
"""
|
||||
# Using early start_date (before test data) so state cursor is used for filtering
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 1, 1, tzinfo=timezone.utc)).build()
|
||||
# Using +0000 format (without colon) to match connector's timezone format
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": "2024-03-01T00:00:00+0000"}).build()
|
||||
|
||||
# Lists stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_old",
|
||||
"attributes": {
|
||||
"name": "Old List",
|
||||
"created": "2024-01-01T10:00:00+00:00",
|
||||
"updated": "2024-02-01T10:00:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
assert len(output.records) == 0
|
||||
|
||||
@HttpMocker()
|
||||
def test_transformation_adds_updated_field(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the AddFields transformation correctly extracts 'updated' from attributes.
|
||||
|
||||
Given: A list record with updated in attributes
|
||||
When: Running a sync
|
||||
Then: The 'updated' field should be added at the root level of the record
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Lists stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_transform_test",
|
||||
"attributes": {
|
||||
"name": "Transform Test",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T14:45:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert "updated" in record
|
||||
assert record["updated"] == "2024-05-31T14:45:00+00:00"
|
||||
|
||||
@HttpMocker()
|
||||
def test_rate_limit_429_handling(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles 429 rate limit responses with RATE_LIMITED action.
|
||||
|
||||
Given: An API that returns a 429 rate limit error
|
||||
When: Making an API request
|
||||
Then: The connector should respect the Retry-After header and retry
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Lists stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
[
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Rate limit exceeded"}]}),
|
||||
status_code=429,
|
||||
headers={"Retry-After": "1"},
|
||||
),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_after_retry",
|
||||
"attributes": {
|
||||
"name": "After Retry",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "list_after_retry"
|
||||
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
# Check for backoff log message pattern
|
||||
assert any(
|
||||
"Backing off" in msg and "UserDefinedBackoffException" in msg and "429" in msg for msg in log_messages
|
||||
), "Expected backoff log message for 429 rate limit"
|
||||
# Check for retry/sleeping log message pattern
|
||||
assert any(
|
||||
"Sleeping for" in msg and "seconds" in msg for msg in log_messages
|
||||
), "Expected retry sleeping log message for 429 rate limit"
|
||||
|
||||
@HttpMocker()
|
||||
def test_unauthorized_401_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 401 Unauthorized errors with FAIL action.
|
||||
|
||||
Given: Invalid API credentials
|
||||
When: Making an API request that returns 401
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key("invalid_key").with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Lists stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint("invalid_key").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Invalid API key"}]}),
|
||||
status_code=401,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 401 authentication failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_forbidden_403_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 403 Forbidden errors with FAIL action.
|
||||
|
||||
The manifest configures 403 errors with action: FAIL, which means the connector
|
||||
should fail the sync when permission errors occur.
|
||||
|
||||
Given: API credentials with insufficient permissions
|
||||
When: Making an API request that returns 403
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Lists stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Forbidden - insufficient permissions"}]}),
|
||||
status_code=403,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 403 permission failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_empty_results(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles empty results gracefully.
|
||||
|
||||
Given: An API that returns no lists
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should return zero records without errors
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Lists stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"data": [], "links": {"self": "https://a.klaviyo.com/api/lists", "next": None}}),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 0
|
||||
assert not any(log.log.level == "ERROR" for log in output.logs)
|
||||
@@ -0,0 +1,653 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest import TestCase
|
||||
|
||||
import freezegun
|
||||
from unit_tests.conftest import get_source
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from airbyte_cdk.test.catalog_builder import CatalogBuilder
|
||||
from airbyte_cdk.test.entrypoint_wrapper import read
|
||||
from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse
|
||||
from airbyte_cdk.test.state_builder import StateBuilder
|
||||
from integration.config import ConfigBuilder
|
||||
from integration.request_builder import KlaviyoRequestBuilder
|
||||
from integration.response_builder import KlaviyoPaginatedResponseBuilder
|
||||
|
||||
|
||||
_NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
_STREAM_NAME = "lists_detailed"
|
||||
_PARENT_STREAM_NAME = "lists"
|
||||
_API_KEY = "test_api_key_abc123"
|
||||
|
||||
|
||||
@freezegun.freeze_time(_NOW.isoformat())
|
||||
class TestListsDetailedStream(TestCase):
|
||||
"""
|
||||
Tests for the Klaviyo 'lists_detailed' stream.
|
||||
|
||||
Stream configuration from manifest.yaml:
|
||||
- Substream of 'lists' stream using SubstreamPartitionRouter
|
||||
- Fetches detailed list information with additional-fields[list]=profile_count
|
||||
- Client-side incremental sync (is_client_side_incremental: true)
|
||||
- DatetimeBasedCursor on 'updated' field
|
||||
- is_data_feed: true
|
||||
- Pagination: CursorPagination
|
||||
- Error handling: 429 RATE_LIMITED, 401/403 FAIL
|
||||
- Transformations: AddFields to extract 'updated' from attributes
|
||||
"""
|
||||
|
||||
@HttpMocker()
|
||||
def test_full_refresh_with_two_parent_records(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that substream correctly fetches data for multiple parent records.
|
||||
|
||||
Given: A parent stream (lists) that returns two list records
|
||||
When: Running a full refresh sync for lists_detailed
|
||||
Then: The connector should fetch detailed data for each parent list
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Parent stream: lists (returns list IDs that become slices for the substream)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{"type": "list", "id": "list_001", "attributes": {"name": "List 1", "updated": "2024-05-31T12:30:00+00:00"}},
|
||||
{"type": "list", "id": "list_002", "attributes": {"name": "List 2", "updated": "2024-05-31T12:30:00+00:00"}},
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
# Substream: lists_detailed for list_001 (calls /api/lists/{list_id} with additional-fields[list]=profile_count)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_detailed_endpoint(_API_KEY, "list_001").with_additional_fields_list("profile_count").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "list",
|
||||
"id": "list_001",
|
||||
"attributes": {
|
||||
"name": "Newsletter Subscribers",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
"profile_count": 1500,
|
||||
},
|
||||
},
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists/list_001"},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
# Substream: lists_detailed for list_002
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_detailed_endpoint(_API_KEY, "list_002").with_additional_fields_list("profile_count").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "list",
|
||||
"id": "list_002",
|
||||
"attributes": {
|
||||
"name": "VIP Customers",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"opt_in_process": "double_opt_in",
|
||||
"profile_count": 500,
|
||||
},
|
||||
},
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists/list_002"},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 2
|
||||
record_ids = [r.record.data["id"] for r in output.records]
|
||||
assert "list_001" in record_ids
|
||||
assert "list_002" in record_ids
|
||||
|
||||
list_001_record = next(r for r in output.records if r.record.data["id"] == "list_001")
|
||||
assert list_001_record.record.data["attributes"]["name"] == "Newsletter Subscribers"
|
||||
assert list_001_record.record.data["attributes"]["profile_count"] == 1500
|
||||
|
||||
list_002_record = next(r for r in output.records if r.record.data["id"] == "list_002")
|
||||
assert list_002_record.record.data["attributes"]["name"] == "VIP Customers"
|
||||
assert list_002_record.record.data["attributes"]["profile_count"] == 500
|
||||
|
||||
@HttpMocker()
|
||||
def test_pagination_multiple_pages_parent_stream(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fetches all pages from parent stream.
|
||||
|
||||
Given: A parent stream (lists) that returns multiple pages
|
||||
When: Running a full refresh sync for lists_detailed
|
||||
Then: The connector should follow pagination and fetch detailed data for all parent records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Parent stream: lists with pagination (returns list IDs across multiple pages)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
[
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[{"type": "list", "id": "list_001", "attributes": {"name": "List 1", "updated": "2024-05-31T10:00:00+00:00"}}]
|
||||
)
|
||||
.with_next_page_link("https://a.klaviyo.com/api/lists?page[cursor]=abc123")
|
||||
.build(),
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[{"type": "list", "id": "list_002", "attributes": {"name": "List 2", "updated": "2024-05-31T11:00:00+00:00"}}]
|
||||
)
|
||||
.build(),
|
||||
],
|
||||
)
|
||||
|
||||
# Substream: lists_detailed for list_001
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_detailed_endpoint(_API_KEY, "list_001").with_additional_fields_list("profile_count").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "list",
|
||||
"id": "list_001",
|
||||
"attributes": {
|
||||
"name": "List 1",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
"profile_count": 100,
|
||||
},
|
||||
},
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists/list_001"},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
# Substream: lists_detailed for list_002
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_detailed_endpoint(_API_KEY, "list_002").with_additional_fields_list("profile_count").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "list",
|
||||
"id": "list_002",
|
||||
"attributes": {
|
||||
"name": "List 2",
|
||||
"created": "2024-05-31T11:00:00+00:00",
|
||||
"updated": "2024-05-31T11:00:00+00:00",
|
||||
"opt_in_process": "double_opt_in",
|
||||
"profile_count": 200,
|
||||
},
|
||||
},
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists/list_002"},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 2
|
||||
assert output.records[0].record.data["id"] == "list_001"
|
||||
assert output.records[1].record.data["id"] == "list_002"
|
||||
|
||||
@HttpMocker()
|
||||
def test_client_side_incremental_first_sync_no_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test first incremental sync with no previous state (client-side incremental).
|
||||
|
||||
Given: No previous state (first sync)
|
||||
When: Running an incremental sync
|
||||
Then: The connector should fetch all records and emit state message
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Parent stream: lists
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{"type": "list", "id": "list_001", "attributes": {"name": "Test List", "updated": "2024-05-31T12:30:00+00:00"}}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
# Substream: lists_detailed for list_001
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_detailed_endpoint(_API_KEY, "list_001").with_additional_fields_list("profile_count").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "list",
|
||||
"id": "list_001",
|
||||
"attributes": {
|
||||
"name": "Test List",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
"profile_count": 1000,
|
||||
},
|
||||
},
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists/list_001"},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "list_001"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
|
||||
@HttpMocker()
|
||||
def test_client_side_incremental_with_prior_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test client-side incremental sync with a prior state from previous sync.
|
||||
|
||||
For client-side incremental streams (is_client_side_incremental: true), the connector
|
||||
skips fetching details for parent records that are older than the effective cursor.
|
||||
The effective cursor is max(start_date, state), so we use an early start_date to ensure
|
||||
the state cursor is used for filtering.
|
||||
|
||||
Given: A previous sync state with an updated cursor value
|
||||
When: Running an incremental sync
|
||||
Then: The connector should skip old records and only fetch details for new/updated records
|
||||
"""
|
||||
# Use early start_date so state cursor (2024-03-01) becomes the effective cursor
|
||||
# Effective cursor = max(start_date, state) = max(2024-01-01, 2024-03-01) = 2024-03-01
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 1, 1, tzinfo=timezone.utc)).build()
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": "2024-03-01T00:00:00+00:00"}).build()
|
||||
|
||||
# Parent stream: lists (returns both old and new list IDs)
|
||||
# The connector will check the updated timestamp and skip fetching details for old records
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{"type": "list", "id": "list_old", "attributes": {"name": "Old List", "updated": "2024-02-15T10:00:00+00:00"}},
|
||||
{"type": "list", "id": "list_new", "attributes": {"name": "New List", "updated": "2024-03-15T10:00:00+00:00"}},
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
# Substream: lists_detailed for list_new only (connector skips list_old because it's older than state cursor)
|
||||
# Use with_any_query_params() because the exact query params may vary
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_detailed_endpoint(_API_KEY, "list_new").with_any_query_params().build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "list",
|
||||
"id": "list_new",
|
||||
"attributes": {
|
||||
"name": "New List",
|
||||
"created": "2024-03-10T10:00:00+00:00",
|
||||
"updated": "2024-03-15T10:00:00+00:00",
|
||||
"opt_in_process": "double_opt_in",
|
||||
"profile_count": 1500,
|
||||
},
|
||||
},
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists/list_new"},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "list_new"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
|
||||
@HttpMocker()
|
||||
def test_transformation_adds_updated_field(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the AddFields transformation correctly extracts 'updated' from attributes.
|
||||
|
||||
Given: A list_detailed record with updated in attributes
|
||||
When: Running a sync
|
||||
Then: The 'updated' field should be added at the root level of the record
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Parent stream: lists
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_transform_test",
|
||||
"attributes": {"name": "Transform Test", "updated": "2024-05-31T14:45:00+00:00"},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
# Substream: lists_detailed for list_transform_test
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_detailed_endpoint(_API_KEY, "list_transform_test")
|
||||
.with_additional_fields_list("profile_count")
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "list",
|
||||
"id": "list_transform_test",
|
||||
"attributes": {
|
||||
"name": "Transform Test",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T14:45:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
"profile_count": 750,
|
||||
},
|
||||
},
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists/list_transform_test"},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert "updated" in record
|
||||
assert record["updated"] == "2024-05-31T14:45:00+00:00"
|
||||
|
||||
@HttpMocker()
|
||||
def test_profile_count_additional_field(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the additional-fields[list]=profile_count parameter returns profile_count.
|
||||
|
||||
The lists_detailed stream requests additional fields to get profile_count.
|
||||
|
||||
Given: An API response with profile_count in attributes
|
||||
When: Running a sync
|
||||
Then: The record should contain the profile_count field
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Parent stream: lists
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_with_count",
|
||||
"attributes": {"name": "List with Profile Count", "updated": "2024-05-31T12:30:00+00:00"},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
# Substream: lists_detailed for list_with_count (includes profile_count via additional-fields)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_detailed_endpoint(_API_KEY, "list_with_count").with_additional_fields_list("profile_count").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "list",
|
||||
"id": "list_with_count",
|
||||
"attributes": {
|
||||
"name": "List with Profile Count",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
"profile_count": 2500,
|
||||
},
|
||||
},
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists/list_with_count"},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert record["attributes"]["profile_count"] == 2500
|
||||
|
||||
@HttpMocker()
|
||||
def test_rate_limit_429_handling(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles 429 rate limit responses with RATE_LIMITED action.
|
||||
|
||||
Given: An API that returns a 429 rate limit error
|
||||
When: Making an API request
|
||||
Then: The connector should respect the Retry-After header and retry
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Parent stream: lists (first returns 429, then success after retry)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
[
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Rate limit exceeded"}]}),
|
||||
status_code=429,
|
||||
headers={"Retry-After": "1"},
|
||||
),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_after_retry",
|
||||
"attributes": {"name": "After Retry", "updated": "2024-05-31T10:00:00+00:00"},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# Substream: lists_detailed for list_after_retry
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_detailed_endpoint(_API_KEY, "list_after_retry")
|
||||
.with_additional_fields_list("profile_count")
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"type": "list",
|
||||
"id": "list_after_retry",
|
||||
"attributes": {
|
||||
"name": "After Retry",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
"profile_count": 100,
|
||||
},
|
||||
},
|
||||
"links": {"self": "https://a.klaviyo.com/api/lists/list_after_retry"},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "list_after_retry"
|
||||
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
# Check for backoff log message pattern
|
||||
assert any(
|
||||
"Backing off" in msg and "UserDefinedBackoffException" in msg and "429" in msg for msg in log_messages
|
||||
), "Expected backoff log message for 429 rate limit"
|
||||
# Check for retry/sleeping log message pattern
|
||||
assert any(
|
||||
"Sleeping for" in msg and "seconds" in msg for msg in log_messages
|
||||
), "Expected retry sleeping log message for 429 rate limit"
|
||||
|
||||
@HttpMocker()
|
||||
def test_unauthorized_401_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 401 Unauthorized errors with FAIL action.
|
||||
|
||||
Given: Invalid API credentials
|
||||
When: Making an API request that returns 401
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key("invalid_key").with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# lists_detailed is a substream of lists. The parent lists stream has no query parameters.
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint("invalid_key").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Invalid API key"}]}),
|
||||
status_code=401,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 401 authentication failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_forbidden_403_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 403 Forbidden errors with FAIL action.
|
||||
|
||||
The manifest configures 403 errors with action: FAIL, which means the connector
|
||||
should fail the sync when permission errors occur.
|
||||
|
||||
Given: API credentials with insufficient permissions
|
||||
When: Making an API request that returns 403
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# lists_detailed is a substream of lists. The parent lists stream has no query parameters.
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Forbidden - insufficient permissions"}]}),
|
||||
status_code=403,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 403 permission failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_empty_parent_stream_results(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles empty parent stream results gracefully.
|
||||
|
||||
Given: A parent stream (lists) that returns no records
|
||||
When: Running a full refresh sync for lists_detailed
|
||||
Then: The connector should return zero records without errors
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# lists_detailed is a substream of lists. The parent lists stream has no query parameters.
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.lists_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"data": [], "links": {"self": "https://a.klaviyo.com/api/lists", "next": None}}),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 0
|
||||
assert not any(log.log.level == "ERROR" for log in output.logs)
|
||||
@@ -0,0 +1,500 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest import TestCase
|
||||
|
||||
import freezegun
|
||||
from unit_tests.conftest import get_source
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from airbyte_cdk.test.catalog_builder import CatalogBuilder
|
||||
from airbyte_cdk.test.entrypoint_wrapper import read
|
||||
from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse
|
||||
from airbyte_cdk.test.state_builder import StateBuilder
|
||||
from integration.config import ConfigBuilder
|
||||
from integration.request_builder import KlaviyoRequestBuilder
|
||||
from integration.response_builder import KlaviyoPaginatedResponseBuilder
|
||||
|
||||
|
||||
_NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
_STREAM_NAME = "metrics"
|
||||
_API_KEY = "test_api_key_abc123"
|
||||
|
||||
|
||||
@freezegun.freeze_time(_NOW.isoformat())
|
||||
class TestMetricsStream(TestCase):
|
||||
"""
|
||||
Tests for the Klaviyo 'metrics' stream.
|
||||
|
||||
Stream configuration from manifest.yaml:
|
||||
- Client-side incremental sync (is_client_side_incremental: true)
|
||||
- DatetimeBasedCursor on 'updated' field
|
||||
- is_data_feed: true - stops pagination when old records are detected
|
||||
- Pagination: CursorPagination
|
||||
- Error handling: 429 RATE_LIMITED, 401/403 FAIL
|
||||
- Transformations: AddFields to extract 'updated' from attributes
|
||||
"""
|
||||
|
||||
@HttpMocker()
|
||||
def test_full_refresh_single_page(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test full refresh sync with a single page of results.
|
||||
|
||||
Given: A configured Klaviyo connector
|
||||
When: Running a full refresh sync for the metrics stream
|
||||
Then: The connector should make the correct API request and return all records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Metrics stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.metrics_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "metric",
|
||||
"id": "metric_001",
|
||||
"attributes": {
|
||||
"name": "Placed Order",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"integration": {"id": "integration_001", "name": "Shopify"},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/metrics", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert record["id"] == "metric_001"
|
||||
assert record["attributes"]["name"] == "Placed Order"
|
||||
|
||||
@HttpMocker()
|
||||
def test_pagination_multiple_pages(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fetches all pages when pagination is present.
|
||||
|
||||
Given: An API that returns multiple pages of metrics
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should follow pagination links and return all records
|
||||
|
||||
Note: Uses with_any_query_params() because pagination adds page[cursor] to the
|
||||
request params, making exact matching impractical.
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Use a single mock with any query params since pagination adds page[cursor]
|
||||
# which makes exact query param matching impractical
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.metrics_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
[
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "metric",
|
||||
"id": "metric_001",
|
||||
"attributes": {
|
||||
"name": "Metric 1",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
"integration": {"id": "int_001", "name": "Shopify"},
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.with_next_page_link("https://a.klaviyo.com/api/metrics?page[cursor]=abc123")
|
||||
.build(),
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "metric",
|
||||
"id": "metric_002",
|
||||
"attributes": {
|
||||
"name": "Metric 2",
|
||||
"created": "2024-05-31T11:00:00+00:00",
|
||||
"updated": "2024-05-31T11:00:00+00:00",
|
||||
"integration": {"id": "int_001", "name": "Shopify"},
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.build(),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 2
|
||||
assert output.records[0].record.data["id"] == "metric_001"
|
||||
assert output.records[1].record.data["id"] == "metric_002"
|
||||
|
||||
@HttpMocker()
|
||||
def test_client_side_incremental_first_sync_no_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test first incremental sync with no previous state (client-side incremental).
|
||||
|
||||
Given: No previous state (first sync)
|
||||
When: Running an incremental sync
|
||||
Then: The connector should fetch all records and emit state message
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Metrics stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.metrics_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "metric",
|
||||
"id": "metric_001",
|
||||
"attributes": {
|
||||
"name": "Test Metric",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
"integration": {"id": "int_001", "name": "Shopify"},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/metrics", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "metric_001"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
latest_state = output.most_recent_state.stream_state.__dict__
|
||||
assert "updated" in latest_state
|
||||
|
||||
@HttpMocker()
|
||||
def test_client_side_incremental_with_prior_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test client-side incremental sync with a prior state from previous sync.
|
||||
|
||||
For client-side incremental streams (is_client_side_incremental: true), the connector
|
||||
fetches all records from the API but filters them client-side based on the state.
|
||||
|
||||
Given: A previous sync state with an updated cursor value
|
||||
When: Running an incremental sync
|
||||
Then: The connector should filter records client-side and only return new/updated records
|
||||
"""
|
||||
# Using early start_date (before test data) so state cursor is used for filtering
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 1, 1, tzinfo=timezone.utc)).build()
|
||||
# Using +0000 format (without colon) to match connector's timezone format
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": "2024-03-01T00:00:00+0000"}).build()
|
||||
|
||||
# Metrics stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.metrics_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "metric",
|
||||
"id": "metric_old",
|
||||
"attributes": {
|
||||
"name": "Old Metric",
|
||||
"created": "2024-01-01T10:00:00+00:00",
|
||||
"updated": "2024-02-15T10:00:00+00:00",
|
||||
"integration": {"id": "int_001", "name": "Shopify"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "metric",
|
||||
"id": "metric_new",
|
||||
"attributes": {
|
||||
"name": "New Metric",
|
||||
"created": "2024-03-10T10:00:00+00:00",
|
||||
"updated": "2024-03-15T10:00:00+00:00",
|
||||
"integration": {"id": "int_001", "name": "Shopify"},
|
||||
},
|
||||
},
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/metrics", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "metric_new"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
latest_state = output.most_recent_state.stream_state.__dict__
|
||||
# Note: The connector returns datetime with +0000 format (without colon)
|
||||
assert latest_state["updated"] == "2024-03-15T10:00:00+0000"
|
||||
|
||||
@HttpMocker()
|
||||
def test_data_feed_stops_pagination_on_old_records(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that pagination stops when old records are detected (is_data_feed: true).
|
||||
|
||||
For data feed streams, if Page 1 contains records older than state, Page 2 should not be fetched.
|
||||
|
||||
Given: A state with a cursor value and API returning old records
|
||||
When: Running an incremental sync
|
||||
Then: The connector should stop pagination when old records are detected
|
||||
"""
|
||||
# Using early start_date (before test data) so state cursor is used for filtering
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 1, 1, tzinfo=timezone.utc)).build()
|
||||
# Using +0000 format (without colon) to match connector's timezone format
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": "2024-03-01T00:00:00+0000"}).build()
|
||||
|
||||
# Metrics stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.metrics_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "metric",
|
||||
"id": "metric_old",
|
||||
"attributes": {
|
||||
"name": "Old Metric",
|
||||
"created": "2024-01-01T10:00:00+00:00",
|
||||
"updated": "2024-02-01T10:00:00+00:00",
|
||||
"integration": {"id": "int_001", "name": "Shopify"},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/metrics", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
assert len(output.records) == 0
|
||||
|
||||
@HttpMocker()
|
||||
def test_transformation_adds_updated_field(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the AddFields transformation correctly extracts 'updated' from attributes.
|
||||
|
||||
Given: A metric record with updated in attributes
|
||||
When: Running a sync
|
||||
Then: The 'updated' field should be added at the root level of the record
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Metrics stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.metrics_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "metric",
|
||||
"id": "metric_transform_test",
|
||||
"attributes": {
|
||||
"name": "Transform Test",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T14:45:00+00:00",
|
||||
"integration": {"id": "int_001", "name": "Shopify"},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/metrics", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert "updated" in record
|
||||
assert record["updated"] == "2024-05-31T14:45:00+00:00"
|
||||
|
||||
@HttpMocker()
|
||||
def test_rate_limit_429_handling(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles 429 rate limit responses with RATE_LIMITED action.
|
||||
|
||||
Given: An API that returns a 429 rate limit error
|
||||
When: Making an API request
|
||||
Then: The connector should respect the Retry-After header and retry
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Metrics stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.metrics_endpoint(_API_KEY).build(),
|
||||
[
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Rate limit exceeded"}]}),
|
||||
status_code=429,
|
||||
headers={"Retry-After": "1"},
|
||||
),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "metric",
|
||||
"id": "metric_after_retry",
|
||||
"attributes": {
|
||||
"name": "After Retry",
|
||||
"created": "2024-05-31T10:00:00+00:00",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
"integration": {"id": "int_001", "name": "Shopify"},
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/metrics", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "metric_after_retry"
|
||||
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
# Check for backoff log message pattern
|
||||
assert any(
|
||||
"Backing off" in msg and "UserDefinedBackoffException" in msg and "429" in msg for msg in log_messages
|
||||
), "Expected backoff log message for 429 rate limit"
|
||||
# Check for retry/sleeping log message pattern
|
||||
assert any(
|
||||
"Sleeping for" in msg and "seconds" in msg for msg in log_messages
|
||||
), "Expected retry sleeping log message for 429 rate limit"
|
||||
|
||||
@HttpMocker()
|
||||
def test_unauthorized_401_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 401 Unauthorized errors with FAIL action.
|
||||
|
||||
Given: Invalid API credentials
|
||||
When: Making an API request that returns 401
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key("invalid_key").with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Metrics stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.metrics_endpoint("invalid_key").build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Invalid API key"}]}),
|
||||
status_code=401,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 401 authentication failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_forbidden_403_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 403 Forbidden errors with FAIL action.
|
||||
|
||||
The manifest configures 403 errors with action: FAIL, which means the connector
|
||||
should fail the sync when permission errors occur.
|
||||
|
||||
Given: API credentials with insufficient permissions
|
||||
When: Making an API request that returns 403
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Metrics stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.metrics_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Forbidden - insufficient permissions"}]}),
|
||||
status_code=403,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 403 permission failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_empty_results(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles empty results gracefully.
|
||||
|
||||
Given: An API that returns no metrics
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should return zero records without errors
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Metrics stream has no query parameters (no request_parameters in manifest)
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.metrics_endpoint(_API_KEY).build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"data": [], "links": {"self": "https://a.klaviyo.com/api/metrics", "next": None}}),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 0
|
||||
assert not any(log.log.level == "ERROR" for log in output.logs)
|
||||
@@ -0,0 +1,594 @@
|
||||
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest import TestCase
|
||||
|
||||
import freezegun
|
||||
from unit_tests.conftest import get_source
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from airbyte_cdk.test.catalog_builder import CatalogBuilder
|
||||
from airbyte_cdk.test.entrypoint_wrapper import read
|
||||
from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse
|
||||
from airbyte_cdk.test.state_builder import StateBuilder
|
||||
from integration.config import ConfigBuilder
|
||||
from integration.request_builder import KlaviyoRequestBuilder
|
||||
from integration.response_builder import KlaviyoPaginatedResponseBuilder
|
||||
|
||||
|
||||
_NOW = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
_STREAM_NAME = "profiles"
|
||||
_API_KEY = "test_api_key_abc123"
|
||||
|
||||
|
||||
@freezegun.freeze_time(_NOW.isoformat())
|
||||
class TestProfilesStream(TestCase):
|
||||
"""
|
||||
Tests for the Klaviyo 'profiles' stream.
|
||||
|
||||
Stream configuration from manifest.yaml:
|
||||
- Incremental sync with DatetimeBasedCursor on 'updated' field
|
||||
- Pagination: CursorPagination with page[size]=100
|
||||
- Error handling: 429 RATE_LIMITED, 401/403 FAIL
|
||||
- Transformations: AddFields to extract 'updated' from attributes
|
||||
"""
|
||||
|
||||
@HttpMocker()
|
||||
def test_full_refresh_single_page(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test full refresh sync with a single page of results.
|
||||
|
||||
Given: A configured Klaviyo connector
|
||||
When: Running a full refresh sync for the profiles stream
|
||||
Then: The connector should make the correct API request and return all records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Validate that the connector sends the correct query parameters
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
"additional-fields[profile]": "predictive_analytics",
|
||||
"page[size]": "100",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_001",
|
||||
"attributes": {
|
||||
"email": "test@example.com",
|
||||
"first_name": "John",
|
||||
"last_name": "Doe",
|
||||
"updated": "2024-01-15T12:30:00+00:00",
|
||||
},
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles/profile_001"},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert record["id"] == "profile_001"
|
||||
assert record["attributes"]["email"] == "test@example.com"
|
||||
assert record["updated"] == "2024-01-15T12:30:00+00:00"
|
||||
|
||||
@HttpMocker()
|
||||
def test_pagination_multiple_pages(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fetches all pages when pagination is present.
|
||||
|
||||
NOTE: This test validates pagination for the 'profiles' stream. All streams
|
||||
in source-klaviyo use the same CursorPagination configuration with RequestPath
|
||||
page_token_option, so this provides pagination coverage for:
|
||||
profiles, global_exclusions, events, events_detailed, email_templates,
|
||||
campaigns, campaigns_detailed, flows, metrics, lists, lists_detailed
|
||||
|
||||
Given: An API that returns multiple pages of profiles
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should follow pagination links and return all records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
# Use a single mock with multiple responses served sequentially.
|
||||
# The first response includes a next_page_link, the second response has no next link.
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY).with_any_query_params().build(),
|
||||
[
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_001",
|
||||
"attributes": {
|
||||
"email": "user1@example.com",
|
||||
"first_name": "User",
|
||||
"last_name": "One",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_002",
|
||||
"attributes": {
|
||||
"email": "user2@example.com",
|
||||
"first_name": "User",
|
||||
"last_name": "Two",
|
||||
"updated": "2024-05-31T11:00:00+00:00",
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
.with_next_page_link("https://a.klaviyo.com/api/profiles?page[cursor]=abc123")
|
||||
.build(),
|
||||
KlaviyoPaginatedResponseBuilder()
|
||||
.with_records(
|
||||
[
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_003",
|
||||
"attributes": {
|
||||
"email": "user3@example.com",
|
||||
"first_name": "User",
|
||||
"last_name": "Three",
|
||||
"updated": "2024-05-31T12:00:00+00:00",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
.build(),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 3
|
||||
assert output.records[0].record.data["id"] == "profile_001"
|
||||
assert output.records[1].record.data["id"] == "profile_002"
|
||||
assert output.records[2].record.data["id"] == "profile_003"
|
||||
assert all(record.record.stream == _STREAM_NAME for record in output.records)
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_first_sync_no_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test first incremental sync with no previous state.
|
||||
|
||||
Given: No previous state (first sync)
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use start_date from config and emit state message
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
"additional-fields[profile]": "predictive_analytics",
|
||||
"page[size]": "100",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_001",
|
||||
"attributes": {
|
||||
"email": "test@example.com",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "profile_001"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
latest_state = output.most_recent_state.stream_state.__dict__
|
||||
assert "updated" in latest_state
|
||||
|
||||
@HttpMocker()
|
||||
def test_incremental_sync_with_prior_state(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test incremental sync with a prior state from previous sync.
|
||||
|
||||
Given: A previous sync state with an updated cursor value
|
||||
When: Running an incremental sync
|
||||
Then: The connector should use the state cursor and return only new/updated records
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": "2024-05-31T00:00:00+00:00"}).build()
|
||||
|
||||
# When state is provided, the filter uses the state cursor value
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
"additional-fields[profile]": "predictive_analytics",
|
||||
"page[size]": "100",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_new",
|
||||
"attributes": {
|
||||
"email": "new@example.com",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config, state=state)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.incremental).build()
|
||||
output = read(source, config=config, catalog=catalog, state=state)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "profile_new"
|
||||
|
||||
assert len(output.state_messages) > 0
|
||||
latest_state = output.most_recent_state.stream_state.__dict__
|
||||
# Note: The connector returns datetime with +0000 format (without colon)
|
||||
assert latest_state["updated"] == "2024-05-31T10:00:00+0000"
|
||||
|
||||
@HttpMocker()
|
||||
def test_transformation_adds_updated_field(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that the AddFields transformation correctly extracts 'updated' from attributes.
|
||||
|
||||
The manifest configures:
|
||||
transformations:
|
||||
- type: AddFields
|
||||
fields:
|
||||
- path: [updated]
|
||||
value: "{{ record.get('attributes', {}).get('updated') }}"
|
||||
|
||||
Given: A profile record with updated in attributes
|
||||
When: Running a sync
|
||||
Then: The 'updated' field should be added at the root level of the record
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
"additional-fields[profile]": "predictive_analytics",
|
||||
"page[size]": "100",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_transform_test",
|
||||
"attributes": {
|
||||
"email": "transform@example.com",
|
||||
"updated": "2024-05-31T14:45:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
record = output.records[0].record.data
|
||||
assert "updated" in record
|
||||
assert record["updated"] == "2024-05-31T14:45:00+00:00"
|
||||
assert record["attributes"]["updated"] == "2024-05-31T14:45:00+00:00"
|
||||
|
||||
@HttpMocker()
|
||||
def test_rate_limit_429_handling(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles 429 rate limit responses with RATE_LIMITED action.
|
||||
|
||||
The manifest configures:
|
||||
response_filters:
|
||||
- type: HttpResponseFilter
|
||||
action: RATE_LIMITED
|
||||
http_codes: [429]
|
||||
|
||||
Given: An API that returns a 429 rate limit error
|
||||
When: Making an API request
|
||||
Then: The connector should respect the Retry-After header and retry
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
"additional-fields[profile]": "predictive_analytics",
|
||||
"page[size]": "100",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
[
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Rate limit exceeded"}]}),
|
||||
status_code=429,
|
||||
headers={"Retry-After": "1"},
|
||||
),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_after_retry",
|
||||
"attributes": {
|
||||
"email": "retry@example.com",
|
||||
"updated": "2024-05-31T10:00:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "profile_after_retry"
|
||||
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
# Check for backoff log message pattern
|
||||
assert any(
|
||||
"Backing off" in msg and "UserDefinedBackoffException" in msg and "429" in msg for msg in log_messages
|
||||
), "Expected backoff log message for 429 rate limit"
|
||||
# Check for retry/sleeping log message pattern
|
||||
assert any(
|
||||
"Sleeping for" in msg and "seconds" in msg for msg in log_messages
|
||||
), "Expected retry sleeping log message for 429 rate limit"
|
||||
|
||||
@HttpMocker()
|
||||
def test_unauthorized_401_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 401 Unauthorized errors with FAIL action.
|
||||
|
||||
The manifest configures:
|
||||
response_filters:
|
||||
- type: HttpResponseFilter
|
||||
action: FAIL
|
||||
http_codes: [401, 403]
|
||||
failure_type: config_error
|
||||
error_message: "Please provide a valid API key..."
|
||||
|
||||
Given: Invalid API credentials
|
||||
When: Making an API request that returns 401
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key("invalid_key").with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint("invalid_key")
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
"additional-fields[profile]": "predictive_analytics",
|
||||
"page[size]": "100",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Invalid API key"}]}),
|
||||
status_code=401,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 401 authentication failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_forbidden_403_error_fails(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector fails on 403 Forbidden errors with FAIL action.
|
||||
|
||||
The manifest configures 403 errors with action: FAIL, which means the connector
|
||||
should fail the sync when permission errors occur.
|
||||
|
||||
Given: API credentials with insufficient permissions
|
||||
When: Making an API request that returns 403
|
||||
Then: The connector should fail with a config error
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
"additional-fields[profile]": "predictive_analytics",
|
||||
"page[size]": "100",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"errors": [{"detail": "Forbidden - insufficient permissions"}]}),
|
||||
status_code=403,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog, expecting_exception=True)
|
||||
|
||||
assert len(output.records) == 0
|
||||
expected_error_message = "Please provide a valid API key and make sure it has permissions to read specified streams."
|
||||
log_messages = [log.log.message for log in output.logs]
|
||||
assert any(
|
||||
expected_error_message in msg for msg in log_messages
|
||||
), f"Expected error message '{expected_error_message}' in logs for 403 permission failure"
|
||||
|
||||
@HttpMocker()
|
||||
def test_empty_results(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that connector handles empty results gracefully.
|
||||
|
||||
Given: An API that returns no profiles
|
||||
When: Running a full refresh sync
|
||||
Then: The connector should return zero records without errors
|
||||
"""
|
||||
config = ConfigBuilder().with_api_key(_API_KEY).with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc)).build()
|
||||
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
"additional-fields[profile]": "predictive_analytics",
|
||||
"page[size]": "100",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps({"data": [], "links": {"self": "https://a.klaviyo.com/api/profiles", "next": None}}),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 0
|
||||
assert not any(log.log.level == "ERROR" for log in output.logs)
|
||||
|
||||
@HttpMocker()
|
||||
def test_predictive_analytics_disabled(self, http_mocker: HttpMocker):
|
||||
"""
|
||||
Test that predictive_analytics field is not requested when disabled.
|
||||
|
||||
The manifest configures:
|
||||
request_parameters:
|
||||
additional-fields[profile]: >-
|
||||
{{ 'predictive_analytics' if not config['disable_fetching_predictive_analytics'] else '' }}
|
||||
|
||||
Given: Config with disable_fetching_predictive_analytics=True
|
||||
When: Running a sync
|
||||
Then: The additional-fields parameter should be empty
|
||||
"""
|
||||
config = (
|
||||
ConfigBuilder()
|
||||
.with_api_key(_API_KEY)
|
||||
.with_start_date(datetime(2024, 5, 31, tzinfo=timezone.utc))
|
||||
.with_disable_fetching_predictive_analytics(True)
|
||||
.build()
|
||||
)
|
||||
|
||||
# When predictive_analytics is disabled, additional-fields[profile] should be empty string
|
||||
http_mocker.get(
|
||||
KlaviyoRequestBuilder.profiles_endpoint(_API_KEY)
|
||||
.with_query_params(
|
||||
{
|
||||
"filter": "greater-than(updated,2024-05-31T00:00:00+0000)",
|
||||
"sort": "updated",
|
||||
"additional-fields[profile]": "",
|
||||
"page[size]": "100",
|
||||
}
|
||||
)
|
||||
.build(),
|
||||
HttpResponse(
|
||||
body=json.dumps(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_no_analytics",
|
||||
"attributes": {
|
||||
"email": "noanalytics@example.com",
|
||||
"updated": "2024-05-31T12:30:00+00:00",
|
||||
},
|
||||
}
|
||||
],
|
||||
"links": {"self": "https://a.klaviyo.com/api/profiles", "next": None},
|
||||
}
|
||||
),
|
||||
status_code=200,
|
||||
),
|
||||
)
|
||||
|
||||
source = get_source(config=config)
|
||||
catalog = CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build()
|
||||
output = read(source, config=config, catalog=catalog)
|
||||
|
||||
assert len(output.records) == 1
|
||||
assert output.records[0].record.data["id"] == "profile_no_analytics"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -10,8 +10,12 @@ authors = ["Airbyte <contact@airbyte.io>"]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.10,<3.13"
|
||||
airbyte-cdk = "^6"
|
||||
airbyte-cdk = "^7"
|
||||
pytest = "^8"
|
||||
freezegun = "^1.4.0"
|
||||
pytest-mock = "^3.6.1"
|
||||
requests-mock = "^1.12.1"
|
||||
mock = "^5.1.0"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
filterwarnings = [
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
[
|
||||
{
|
||||
"type": "campaign",
|
||||
"id": "campaign_001",
|
||||
"attributes": {
|
||||
"name": "Summer Sale Campaign",
|
||||
"status": "Sent",
|
||||
"archived": false,
|
||||
"audiences": {
|
||||
"included": ["list_001"],
|
||||
"excluded": []
|
||||
},
|
||||
"send_options": {
|
||||
"use_smart_sending": true,
|
||||
"is_transactional": false
|
||||
},
|
||||
"tracking_options": {
|
||||
"is_add_utm": true,
|
||||
"is_tracking_clicks": true,
|
||||
"is_tracking_opens": true
|
||||
},
|
||||
"send_strategy": {
|
||||
"method": "immediate",
|
||||
"options_static": null,
|
||||
"options_throttled": null,
|
||||
"options_sto": null
|
||||
},
|
||||
"created_at": "2024-01-01T10:00:00+00:00",
|
||||
"scheduled_at": "2024-01-15T10:00:00+00:00",
|
||||
"updated_at": "2024-01-15T12:30:00+00:00",
|
||||
"send_time": "2024-01-15T10:00:00+00:00"
|
||||
},
|
||||
"relationships": {
|
||||
"campaign-messages": {
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/campaigns/campaign_001/relationships/campaign-messages",
|
||||
"related": "https://a.klaviyo.com/api/campaigns/campaign_001/campaign-messages"
|
||||
}
|
||||
},
|
||||
"tags": {
|
||||
"data": []
|
||||
}
|
||||
},
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/campaigns/campaign_001"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,17 @@
|
||||
[
|
||||
{
|
||||
"type": "template",
|
||||
"id": "template_001",
|
||||
"attributes": {
|
||||
"name": "Welcome Email",
|
||||
"editor_type": "DRAG_AND_DROP",
|
||||
"html": "<html><body><h1>Welcome!</h1></body></html>",
|
||||
"text": "Welcome!",
|
||||
"created": "2024-01-01T10:00:00+00:00",
|
||||
"updated": "2024-01-15T12:30:00+00:00"
|
||||
},
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/templates/template_001"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,42 @@
|
||||
[
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_001",
|
||||
"attributes": {
|
||||
"timestamp": "2024-01-15T10:30:00+00:00",
|
||||
"datetime": "2024-01-15T10:30:00+00:00",
|
||||
"uuid": "550e8400-e29b-41d4-a716-446655440000",
|
||||
"event_properties": {
|
||||
"value": 99.99,
|
||||
"currency": "USD",
|
||||
"items": [
|
||||
{
|
||||
"product_id": "prod_001",
|
||||
"product_name": "Test Product",
|
||||
"quantity": 2,
|
||||
"price": 49.99
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"relationships": {
|
||||
"metric": {
|
||||
"data": {
|
||||
"type": "metric",
|
||||
"id": "metric_001"
|
||||
}
|
||||
},
|
||||
"attributions": {
|
||||
"data": [
|
||||
{
|
||||
"type": "attribution",
|
||||
"id": "attr_001"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/events/event_001"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,49 @@
|
||||
[
|
||||
{
|
||||
"type": "event",
|
||||
"id": "event_detailed_001",
|
||||
"attributes": {
|
||||
"timestamp": "2024-01-15T10:30:00+00:00",
|
||||
"datetime": "2024-01-15T10:30:00+00:00",
|
||||
"uuid": "550e8400-e29b-41d4-a716-446655440001",
|
||||
"event_properties": {
|
||||
"value": 149.99,
|
||||
"currency": "USD",
|
||||
"order_id": "order_001",
|
||||
"items": [
|
||||
{
|
||||
"product_id": "prod_001",
|
||||
"product_name": "Premium Product",
|
||||
"quantity": 1,
|
||||
"price": 149.99
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"relationships": {
|
||||
"metric": {
|
||||
"data": {
|
||||
"type": "metric",
|
||||
"id": "metric_001"
|
||||
}
|
||||
},
|
||||
"attributions": {
|
||||
"data": [
|
||||
{
|
||||
"type": "attribution",
|
||||
"id": "attr_001"
|
||||
}
|
||||
]
|
||||
},
|
||||
"profile": {
|
||||
"data": {
|
||||
"type": "profile",
|
||||
"id": "profile_001"
|
||||
}
|
||||
}
|
||||
},
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/events/event_detailed_001"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,28 @@
|
||||
[
|
||||
{
|
||||
"type": "flow",
|
||||
"id": "flow_001",
|
||||
"attributes": {
|
||||
"name": "Welcome Series",
|
||||
"status": "live",
|
||||
"archived": false,
|
||||
"created": "2024-01-01T10:00:00+00:00",
|
||||
"updated": "2024-01-15T12:30:00+00:00",
|
||||
"trigger_type": "List"
|
||||
},
|
||||
"relationships": {
|
||||
"flow-actions": {
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/flows/flow_001/relationships/flow-actions",
|
||||
"related": "https://a.klaviyo.com/api/flows/flow_001/flow-actions"
|
||||
}
|
||||
},
|
||||
"tags": {
|
||||
"data": []
|
||||
}
|
||||
},
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/flows/flow_001"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,40 @@
|
||||
[
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_excluded_001",
|
||||
"attributes": {
|
||||
"email": "excluded@example.com",
|
||||
"phone_number": "+1234567890",
|
||||
"external_id": "ext_excluded_001",
|
||||
"first_name": "Jane",
|
||||
"last_name": "Smith",
|
||||
"organization": "Excluded Corp",
|
||||
"locale": "en-US",
|
||||
"created": "2024-01-01T10:00:00+00:00",
|
||||
"updated": "2024-01-15T12:30:00+00:00",
|
||||
"subscriptions": {
|
||||
"email": {
|
||||
"marketing": {
|
||||
"can_receive_email_marketing": false,
|
||||
"consent": "UNSUBSCRIBED",
|
||||
"consent_timestamp": "2024-01-10T10:00:00+00:00",
|
||||
"suppression": [
|
||||
{
|
||||
"reason": "USER_SUPPRESSED",
|
||||
"timestamp": "2024-01-10T10:00:00+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"sms": {
|
||||
"marketing": {
|
||||
"can_receive_sms_marketing": false
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/profiles/profile_excluded_001"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,15 @@
|
||||
[
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_001",
|
||||
"attributes": {
|
||||
"name": "Newsletter Subscribers",
|
||||
"created": "2024-01-01T10:00:00+00:00",
|
||||
"updated": "2024-01-15T12:30:00+00:00",
|
||||
"opt_in_process": "single_opt_in"
|
||||
},
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/lists/list_001"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"type": "list",
|
||||
"id": "list_001",
|
||||
"attributes": {
|
||||
"name": "Newsletter Subscribers",
|
||||
"created": "2024-01-01T10:00:00+00:00",
|
||||
"updated": "2024-01-15T12:30:00+00:00",
|
||||
"opt_in_process": "single_opt_in",
|
||||
"profile_count": 1500
|
||||
},
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/lists/list_001"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
[
|
||||
{
|
||||
"type": "metric",
|
||||
"id": "metric_001",
|
||||
"attributes": {
|
||||
"name": "Placed Order",
|
||||
"created": "2024-01-01T10:00:00+00:00",
|
||||
"updated": "2024-01-15T12:30:00+00:00",
|
||||
"integration": {
|
||||
"id": "integration_001",
|
||||
"name": "Shopify",
|
||||
"category": "ecommerce"
|
||||
}
|
||||
},
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/metrics/metric_001"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,63 @@
|
||||
[
|
||||
{
|
||||
"type": "profile",
|
||||
"id": "profile_001",
|
||||
"attributes": {
|
||||
"email": "test@example.com",
|
||||
"phone_number": "+1234567890",
|
||||
"external_id": "ext_001",
|
||||
"first_name": "John",
|
||||
"last_name": "Doe",
|
||||
"organization": "Test Corp",
|
||||
"locale": "en-US",
|
||||
"title": "Engineer",
|
||||
"image": "https://example.com/image.jpg",
|
||||
"created": "2024-01-01T10:00:00+00:00",
|
||||
"updated": "2024-01-15T12:30:00+00:00",
|
||||
"last_event_date": "2024-01-14T08:00:00+00:00",
|
||||
"location": {
|
||||
"address1": "123 Main St",
|
||||
"address2": "Suite 100",
|
||||
"city": "San Francisco",
|
||||
"country": "United States",
|
||||
"latitude": 37.7749,
|
||||
"longitude": -122.4194,
|
||||
"region": "CA",
|
||||
"zip": "94102",
|
||||
"timezone": "America/Los_Angeles",
|
||||
"ip": "192.168.1.1"
|
||||
},
|
||||
"properties": {
|
||||
"custom_field": "custom_value"
|
||||
},
|
||||
"subscriptions": {
|
||||
"email": {
|
||||
"marketing": {
|
||||
"can_receive_email_marketing": true,
|
||||
"consent": "SUBSCRIBED",
|
||||
"consent_timestamp": "2024-01-01T10:00:00+00:00"
|
||||
}
|
||||
},
|
||||
"sms": {
|
||||
"marketing": {
|
||||
"can_receive_sms_marketing": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"predictive_analytics": {
|
||||
"historic_clv": 150.0,
|
||||
"predicted_clv": 200.0,
|
||||
"total_clv": 350.0,
|
||||
"historic_number_of_orders": 5,
|
||||
"predicted_number_of_orders": 3,
|
||||
"average_days_between_orders": 30,
|
||||
"average_order_value": 50.0,
|
||||
"churn_probability": 0.15,
|
||||
"expected_date_of_next_order": "2024-02-15T00:00:00+00:00"
|
||||
}
|
||||
},
|
||||
"links": {
|
||||
"self": "https://a.klaviyo.com/api/profiles/profile_001"
|
||||
}
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user