jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

🐛 Source Outreach: Remove custom incremental class and use new filter syntax (#55888)

Browse Source
This commit is contained in:
kyleromines
2025-03-21 07:35:24 -04:00
committed by GitHub
parent 48dd8bd65e
commit ce53dfa453
4 changed files with 124 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
airbyte-integrations/connectors/source-outreach/components.py
View File

@@ -35,18 +35,3 @@ class CustomExtractor(RecordExtractor):
extracted_records.append(extracted_record)
return extracted_records
@dataclass
class CustomIncrementalSync(DatetimeBasedCursor):
def get_request_params(
self,
*,
stream_state: Optional[StreamState] = None,
stream_slice: Optional[StreamSlice] = None,
next_page_token: Optional[Mapping[str, Any]] = None,
) -> MutableMapping[str, Any]:
params = {}
if self.cursor_field.string in stream_state:
params[f"filter[{self.cursor_field.string}]"] = stream_state[self.cursor_field.string] + "..inf"
return params

175
airbyte-integrations/connectors/source-outreach/manifest.yaml
View File

@@ -24,6 +24,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -38,15 +39,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -66,6 +70,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -80,15 +85,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -108,6 +116,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -122,15 +131,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -150,6 +162,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -164,15 +177,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -192,6 +208,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "100"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -206,15 +223,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -234,6 +254,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -248,15 +269,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -276,6 +300,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -290,15 +315,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -318,6 +346,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -332,15 +361,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -360,6 +392,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -374,15 +407,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -402,6 +438,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -416,15 +453,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -444,6 +484,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -458,15 +499,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -486,6 +530,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -500,15 +545,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -528,6 +576,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -542,15 +591,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -570,6 +622,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -584,15 +637,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -612,6 +668,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -626,15 +683,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -654,6 +714,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -668,15 +729,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -696,6 +760,7 @@ definitions:
count: "false"
sort: updatedAt
page[size]: "1000"
newFilterSyntax: "true"
record_selector:
type: RecordSelector
extractor:
@@ -710,15 +775,18 @@ definitions:
cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
incremental_sync:
type: CustomIncrementalSync
class_name: source_declarative_manifest.components.CustomIncrementalSync
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
type: DatetimeBasedCursor
datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ"
cursor_field: updatedAt
is_data_feed: true
start_datetime:
type: MinMaxDatetime
datetime: "{{ config['start_date'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
start_time_option:
type: RequestOption
field_name: "filter[updatedAt][gte]"
inject_into: "request_parameter"
schema_loader:
type: InlineSchemaLoader
schema:
@@ -794,11 +862,12 @@ spec:
type: string
description: >-
The date from which you'd like to replicate data for Outreach API, in
the format YYYY-MM-DDT00:00:00Z. All data generated after this date
the format YYYY-MM-DDT00:00:00.000Z. All data generated after this date
will be replicated.
title: Start Date
format: date-time
examples:
- "2020-11-16T00:00:00Z"
- "2020-11-16T00:00:00.000Z"
pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$
order: 4
additionalProperties: true

2
airbyte-integrations/connectors/source-outreach/metadata.yaml
View File

@@ -28,7 +28,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: 3490c201-5d95-4783-b600-eaf07a4c7787
dockerImageTag: 1.1.6
dockerImageTag: 1.1.7
dockerRepository: airbyte/source-outreach
githubIssueLabel: source-outreach
icon: outreach.svg

1
docs/integrations/sources/outreach.md
View File

@@ -56,6 +56,7 @@ List of available streams:
| Version | Date | Pull Request | Subject |
| :------ |:-----------| :----- | :------ |
| 1.1.7 | 2025-03-17 | [55888](https://github.com/airbytehq/airbyte/pull/55888) | Remove custom incremental class and use new filter syntax |
| 1.1.6 | 2025-03-09 | [55180](https://github.com/airbytehq/airbyte/pull/55180) | 🐛 Source Outreach: remove stream_state interpolation |
| 1.1.5 | 2025-03-08 | [55561](https://github.com/airbytehq/airbyte/pull/55561) | Update dependencies |
| 1.1.4 | 2025-03-01 | [54611](https://github.com/airbytehq/airbyte/pull/54611) | Update dependencies |