🐛Source Klaviyo: add state_checkpoint_interval (#32291)
This commit is contained in:
Anton Karpets
@@ -8,7 +8,7 @@ data:
|
||||
definitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde
|
||||
connectorBuildOptions:
|
||||
baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c
|
||||
dockerImageTag: 2.0.0
|
||||
dockerImageTag: 2.0.1
|
||||
dockerRepository: airbyte/source-klaviyo
|
||||
githubIssueLabel: source-klaviyo
|
||||
icon: klaviyo.svg
|
||||
|
||||
@@ -169,7 +169,7 @@ class SemiIncrementalKlaviyoStream(KlaviyoStream, ABC):
|
||||
stream_state: Optional[Mapping[str, Any]] = None,
|
||||
) -> Iterable[StreamData]:
|
||||
stream_state = stream_state or {}
|
||||
starting_point = stream_state.get(self.cursor_field) or self._start_ts
|
||||
starting_point = stream_state.get(self.cursor_field, self._start_ts)
|
||||
for record in super().read_records(
|
||||
sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state
|
||||
):
|
||||
@@ -237,11 +237,12 @@ class ArchivedRecordsMixin(IncrementalKlaviyoStream, ABC):
|
||||
|
||||
|
||||
class Profiles(IncrementalKlaviyoStream):
|
||||
"""Docs: https://developers.klaviyo.com/en/reference/get_profiles"""
|
||||
"""Docs: https://developers.klaviyo.com/en/v2023-02-22/reference/get_profiles"""
|
||||
|
||||
cursor_field = "updated"
|
||||
api_revision = "2023-02-22"
|
||||
page_size = 100
|
||||
state_checkpoint_interval = 100 # API can return maximum 100 records per page
|
||||
|
||||
def path(self, *args, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs) -> str:
|
||||
return "profiles"
|
||||
@@ -269,7 +270,7 @@ class Lists(SemiIncrementalKlaviyoStream):
|
||||
|
||||
class GlobalExclusions(Profiles):
|
||||
"""
|
||||
Docs: https://developers.klaviyo.com/en/reference/get_profiles
|
||||
Docs: https://developers.klaviyo.com/en/v2023-02-22/reference/get_profiles
|
||||
This stream takes data from 'profiles' endpoint, but suppressed records only
|
||||
"""
|
||||
|
||||
@@ -293,6 +294,7 @@ class Events(IncrementalKlaviyoStream):
|
||||
"""Docs: https://developers.klaviyo.com/en/reference/get_events"""
|
||||
|
||||
cursor_field = "datetime"
|
||||
state_checkpoint_interval = 200 # API can return maximum 200 records per page
|
||||
|
||||
def path(self, **kwargs) -> str:
|
||||
return "events"
|
||||
@@ -302,6 +304,7 @@ class Flows(ArchivedRecordsMixin, IncrementalKlaviyoStream):
|
||||
"""Docs: https://developers.klaviyo.com/en/reference/get_flows"""
|
||||
|
||||
cursor_field = "updated"
|
||||
state_checkpoint_interval = 50 # API can return maximum 50 records per page
|
||||
|
||||
def path(self, **kwargs) -> str:
|
||||
return "flows"
|
||||
@@ -311,6 +314,7 @@ class EmailTemplates(IncrementalKlaviyoStream):
|
||||
"""Docs: https://developers.klaviyo.com/en/reference/get_templates"""
|
||||
|
||||
cursor_field = "updated"
|
||||
state_checkpoint_interval = 10 # API can return maximum 10 records per page
|
||||
|
||||
def path(self, **kwargs) -> str:
|
||||
return "templates"
|
||||
|
||||
@@ -12,12 +12,12 @@ This page contains the setup guide and reference information for the Klaviyo sou
|
||||
### Step 1: Set up Klaviyo
|
||||
|
||||
1. Create a [Klaviyo account](https://www.klaviyo.com)
|
||||
2. Create a [Private API key](https://help.klaviyo.com/hc/en-us/articles/115005062267-How-to-Manage-Your-Account-s-API-Keys#your-private-api-keys3). Make sure you selected all [scopes](https://help.klaviyo.com/hc/en-us/articles/7423954176283) corresponding to the streams you would like to replicate.
|
||||
2. Create a [Private API key](https://help.klaviyo.com/hc/en-us/articles/115005062267-How-to-Manage-Your-Account-s-API-Keys#your-private-api-keys3). Make sure you selected all [scopes](https://help.klaviyo.com/hc/en-us/articles/7423954176283) corresponding to the streams you would like to replicate. You can find which scope is required for a specific stream by navigating to the relevant API documentation for the streams Airbyte supports.
|
||||
|
||||
### Step 2: Set up the Klaviyo connector in Airbyte
|
||||
|
||||
1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account.
|
||||
2. Click **Sources** and then click **+ New source**.
|
||||
2. Click **Sources** and then click **+ new source**.
|
||||
3. On the Set up the source page, select **Klaviyo** from the **Source type** dropdown.
|
||||
4. Enter a name for the Klaviyo connector.
|
||||
5. For **Api Key**, enter the Klaviyo [Private API key](https://help.klaviyo.com/hc/en-us/articles/115005062267-How-to-Manage-Your-Account-s-API-Keys#your-private-api-keys3).
|
||||
@@ -35,13 +35,14 @@ The Klaviyo source connector supports the following [sync modes](https://docs.ai
|
||||
|
||||
## Supported Streams
|
||||
|
||||
- [Campaigns](https://developers.klaviyo.com/en/v1-2/reference/get-campaigns#get-campaigns)
|
||||
- [Events](https://developers.klaviyo.com/en/v1-2/reference/metrics-timeline)
|
||||
- [GlobalExclusions](https://developers.klaviyo.com/en/v1-2/reference/get-global-exclusions)
|
||||
- [Lists](https://developers.klaviyo.com/en/v1-2/reference/get-lists)
|
||||
- [Metrics](https://developers.klaviyo.com/en/v1-2/reference/get-metrics)
|
||||
- [Campaigns](https://developers.klaviyo.com/en/v2023-06-15/reference/get_campaigns)
|
||||
- [Email Templates](https://developers.klaviyo.com/en/reference/get_templates)
|
||||
- [Events](https://developers.klaviyo.com/en/reference/get_events)
|
||||
- [Flows](https://developers.klaviyo.com/en/reference/get_flows)
|
||||
- [Profiles](https://developers.klaviyo.com/en/reference/get_profiles)
|
||||
- [GlobalExclusions](https://developers.klaviyo.com/en/v2023-02-22/reference/get_profiles)
|
||||
- [Lists](https://developers.klaviyo.com/en/reference/get_lists)
|
||||
- [Metrics](https://developers.klaviyo.com/en/reference/get_metrics)
|
||||
- [Profiles](https://developers.klaviyo.com/en/v2023-02-22/reference/get_profiles)
|
||||
|
||||
## Performance considerations
|
||||
|
||||
@@ -62,6 +63,7 @@ The Klaviyo connector should not run into Klaviyo API limitations under normal u
|
||||
|
||||
| Version | Date | Pull Request | Subject |
|
||||
|:---------|:-----------| :--------------------------------------------------------- |:------------------------------------------------------------------------------------------|
|
||||
| `2.0.1` | 2023-11-08 | [32291](https://github.com/airbytehq/airbyte/pull/32291) | Add logic to have regular checkpointing schedule |
|
||||
| `2.0.0` | 2023-11-03 | [32128](https://github.com/airbytehq/airbyte/pull/32128) | Use the latest API for streams `campaigns`, `email_templates`, `events`, `flows`, `global_exclusions`, `lists`, and `metrics`|
|
||||
| `1.1.0` | 2023-10-23 | [31710](https://github.com/airbytehq/airbyte/pull/31710) | Make `start_date` config field optional |
|
||||
| `1.0.0` | 2023-10-18 | [31565](https://github.com/airbytehq/airbyte/pull/31565) | added new known fields for 'events' stream |
|
||||
|
||||
Reference in New Issue
Block a user