307 lines
11 KiB
YAML
307 lines
11 KiB
YAML
version: "0.1.0"
|
|
|
|
definitions:
|
|
|
|
## bases
|
|
schema_loader:
|
|
type: JsonFileSchemaLoader
|
|
file_path: "./source_intercom/schemas/{{ parameters['name'] }}.json"
|
|
selector:
|
|
description: "Base records selector for Full Refresh streams"
|
|
extractor:
|
|
type: DpathExtractor
|
|
field_path: ["{{ parameters.get('data_field', 'data')}}"]
|
|
requester:
|
|
description: "Base Requester for Full Refresh streams"
|
|
type: CustomRequester
|
|
class_name: source_intercom.components.HttpRequesterWithRateLimiter
|
|
url_base: "https://api.intercom.io/"
|
|
http_method: "GET"
|
|
authenticator:
|
|
type: BearerAuthenticator
|
|
api_token: "{{ config['access_token'] }}"
|
|
request_headers:
|
|
Intercom-Version: '2.5' # ATTENTION: API version change is possible here
|
|
Accept: 'application/json'
|
|
retriever:
|
|
description: "Base Retriever for Full Refresh streams"
|
|
record_selector:
|
|
$ref: "#/definitions/selector"
|
|
requester:
|
|
$ref: "#/definitions/requester"
|
|
paginator:
|
|
type: "DefaultPaginator"
|
|
url_base: "#/definitions/requester/url_base"
|
|
pagination_strategy:
|
|
type: "CursorPagination"
|
|
cursor_value: "{{ response.get('pages', {}).get('next') }}"
|
|
stop_condition: "{{ 'next' not in response.get('pages', {}) }}"
|
|
page_size: 150
|
|
page_size_option:
|
|
inject_into: request_parameter
|
|
field_name: per_page
|
|
page_token_option:
|
|
type: RequestPath
|
|
requester_incremental_search:
|
|
$ref: "#/definitions/requester"
|
|
http_method: "POST"
|
|
request_body_json:
|
|
query: "{
|
|
'operator': 'OR',
|
|
'value': [
|
|
{
|
|
'field': 'updated_at',
|
|
'operator': '>',
|
|
'value': {{ stream_slice.get('prior_state', stream_state.get('prior_state', {})).get('updated_at') or format_datetime(config['start_date'], '%s') }}
|
|
},
|
|
{
|
|
'field': 'updated_at',
|
|
'operator': '=',
|
|
'value': {{ stream_slice.get('prior_state', stream_state.get('prior_state', {})).get('updated_at') or format_datetime(config['start_date'], '%s') }}
|
|
},
|
|
],
|
|
}"
|
|
sort: "{'field': 'updated_at', 'order': 'ascending'}"
|
|
pagination: "{
|
|
'per_page': {{ parameters.get('page_size') }},
|
|
'page': {{ next_page_token.get('next_page_token').get('page') }},
|
|
'starting_after': '{{ next_page_token.get('next_page_token').get('starting_after') }}'
|
|
}"
|
|
|
|
## streams
|
|
# full-refresh
|
|
stream_full_refresh:
|
|
schema_loader:
|
|
$ref: "#/definitions/schema_loader"
|
|
retriever:
|
|
$ref: "#/definitions/retriever"
|
|
admins:
|
|
description: "https://developers.intercom.com/intercom-api-reference/reference#list-admins"
|
|
$ref: "#/definitions/stream_full_refresh"
|
|
$parameters:
|
|
name: "admins"
|
|
primary_key: "id"
|
|
path: "admins"
|
|
data_field: "admins"
|
|
tags:
|
|
description: "https://developers.intercom.com/intercom-api-reference/reference#list-tags-for-an-app"
|
|
$ref: "#/definitions/stream_full_refresh"
|
|
$parameters:
|
|
name: "tags"
|
|
primary_key: "name"
|
|
path: "tags"
|
|
teams:
|
|
description: "https://developers.intercom.com/intercom-api-reference/reference#list-teams"
|
|
$ref: "#/definitions/stream_full_refresh"
|
|
$parameters:
|
|
name: "teams"
|
|
primary_key: "name"
|
|
path: "teams"
|
|
data_field: "teams"
|
|
|
|
stream_data_attributes:
|
|
description: "https://developers.intercom.com/intercom-api-reference/reference#list-data-attributes"
|
|
$ref: "#/definitions/stream_full_refresh"
|
|
retriever:
|
|
$ref: "#/definitions/retriever"
|
|
requester:
|
|
$ref: "#/definitions/requester"
|
|
request_parameters:
|
|
model: "{{ parameters.get('model') }}"
|
|
company_attributes:
|
|
description: "https://developers.intercom.com/intercom-api-reference/reference#list-data-attributes"
|
|
$ref: "#/definitions/stream_data_attributes"
|
|
$parameters:
|
|
name: "company_attributes"
|
|
primary_key: "name"
|
|
path: "data_attributes"
|
|
model: "company"
|
|
contact_attributes:
|
|
description: "https://developers.intercom.com/intercom-api-reference/reference#list-data-attributes"
|
|
$ref: "#/definitions/stream_data_attributes"
|
|
$parameters:
|
|
name: "contact_attributes"
|
|
primary_key: "name"
|
|
path: "data_attributes"
|
|
model: "contact"
|
|
|
|
# semi-incremental
|
|
# (full-refresh and emit records >= *prior state)
|
|
# (prior state - frozen state from previous sync, it automatically updates with next sync)
|
|
stream_semi_incremental:
|
|
$ref: "#/definitions/stream_full_refresh"
|
|
incremental_sync:
|
|
type: CustomIncrementalSync
|
|
class_name: source_intercom.components.IncrementalSingleSlice
|
|
cursor_field: "updated_at"
|
|
retriever:
|
|
$ref: "#/definitions/stream_full_refresh/retriever"
|
|
record_selector:
|
|
$ref: "#/definitions/selector"
|
|
record_filter:
|
|
condition: "{{ record['updated_at'] >= ( stream_state.get('prior_state', {}).get('updated_at', 0) if stream_state else stream_slice.get('prior_state', {}).get('updated_at', 0) ) }}"
|
|
segments:
|
|
description: "https://developers.intercom.com/intercom-api-reference/reference#list-segments"
|
|
$ref: "#/definitions/stream_semi_incremental"
|
|
$parameters:
|
|
name: "segments"
|
|
primary_key: "id"
|
|
path: "segments"
|
|
data_field: "segments"
|
|
companies:
|
|
description: "https://developers.intercom.com/intercom-api-reference/reference/scroll-over-all-companies"
|
|
$ref: "#/definitions/stream_semi_incremental"
|
|
$parameters:
|
|
name: "companies"
|
|
primary_key: "id"
|
|
path: "companies/scroll"
|
|
retriever:
|
|
$ref: "#/definitions/stream_semi_incremental/retriever"
|
|
paginator:
|
|
type: "DefaultPaginator"
|
|
url_base: "#/definitions/requester/url_base"
|
|
pagination_strategy:
|
|
type: "CursorPagination"
|
|
cursor_value: "{{ response.get('scroll_param') }}"
|
|
stop_condition: "{{ not response.get('data') }}"
|
|
page_size: 150
|
|
page_size_option:
|
|
inject_into: request_parameter
|
|
field_name: per_page
|
|
page_token_option:
|
|
type: RequestOption
|
|
field_name: scroll_param
|
|
inject_into: request_parameter
|
|
requester:
|
|
$ref: "#/definitions/requester"
|
|
error_handler:
|
|
type: CompositeErrorHandler
|
|
error_handlers:
|
|
- type: DefaultErrorHandler
|
|
description: "
|
|
400 - existing scroll_param, need to wait at least 60 sec to continue and retry
|
|
500 - server-side error, should retry after 60 sec.
|
|
"
|
|
response_filters:
|
|
- http_codes: [ 400, 500 ]
|
|
action: RETRY
|
|
backoff_strategies:
|
|
- type: ConstantBackoffStrategy
|
|
backoff_time_in_seconds: 60
|
|
- type: DefaultErrorHandler
|
|
description: "404 - scroll_param is expired or not found while requesting, ignore"
|
|
response_filters:
|
|
- http_codes: [ 404 ]
|
|
action: IGNORE
|
|
|
|
# semi-incremental substreams
|
|
substream_semi_incremental:
|
|
$ref: "#/definitions/stream_full_refresh"
|
|
incremental_sync:
|
|
type: CustomIncrementalSync
|
|
class_name: source_intercom.components.IncrementalSubstreamSlicer
|
|
cursor_field: "updated_at"
|
|
retriever:
|
|
$ref: "#/definitions/stream_full_refresh/retriever"
|
|
paginator:
|
|
type: "NoPagination"
|
|
record_selector:
|
|
$ref: "#/definitions/selector"
|
|
record_filter:
|
|
condition: "{{ record['updated_at'] >= stream_state.get('prior_state', {}).get('updated_at', 0) }}"
|
|
conversation_parts:
|
|
$ref: "#/definitions/substream_semi_incremental"
|
|
incremental_sync:
|
|
$ref: "#/definitions/substream_semi_incremental/incremental_sync"
|
|
parent_stream_configs:
|
|
- type: ParentStreamConfig
|
|
stream: "#/definitions/conversations"
|
|
parent_key: "id"
|
|
partition_field: "id"
|
|
$parameters:
|
|
name: "conversation_parts"
|
|
primary_key: "id"
|
|
path: "/conversations/{{ stream_slice.id }}"
|
|
transformations:
|
|
- type: AddFields
|
|
fields:
|
|
- path: ["conversation_id"]
|
|
value: "'{{ stream_slice.id }}'"
|
|
retriever:
|
|
$ref: "#/definitions/substream_semi_incremental/retriever"
|
|
record_selector:
|
|
$ref: "#/definitions/substream_semi_incremental/retriever/record_selector"
|
|
extractor:
|
|
field_path: ["conversation_parts", "conversation_parts"]
|
|
company_segments:
|
|
$ref: "#/definitions/substream_semi_incremental"
|
|
$parameters:
|
|
name: "company_segments"
|
|
primary_key: "id"
|
|
path: "/companies/{{ stream_slice.id }}/segments"
|
|
incremental_sync:
|
|
$ref: "#/definitions/substream_semi_incremental/incremental_sync"
|
|
parent_complete_fetch: true
|
|
parent_stream_configs:
|
|
- type: ParentStreamConfig
|
|
stream: "#/definitions/companies"
|
|
parent_key: "id"
|
|
partition_field: "id"
|
|
retriever:
|
|
$ref: "#/definitions/substream_semi_incremental/retriever"
|
|
|
|
# incremental search
|
|
stream_incremental_search:
|
|
description: "https://developers.intercom.com/intercom-api-reference/reference/pagination-sorting-search"
|
|
$ref: "#/definitions/stream_full_refresh"
|
|
incremental_sync:
|
|
type: CustomIncrementalSync
|
|
class_name: source_intercom.components.IncrementalSingleSlice
|
|
cursor_field: "updated_at"
|
|
retriever:
|
|
$ref: "#/definitions/stream_full_refresh/retriever"
|
|
requester:
|
|
$ref: "#/definitions/requester_incremental_search"
|
|
record_selector:
|
|
$ref: "#/definitions/selector"
|
|
record_filter:
|
|
description: "https://developers.intercom.com/intercom-api-reference/reference/pagination-sorting-search#pagination"
|
|
condition: "{{ record['updated_at'] >= ( stream_state.get('prior_state', {}).get('updated_at', 0) if stream_state else stream_slice.get('prior_state', {}).get('updated_at', 0) ) }}"
|
|
paginator:
|
|
type: "DefaultPaginator"
|
|
url_base: "#/definitions/requester/url_base"
|
|
pagination_strategy:
|
|
type: "CursorPagination"
|
|
cursor_value: "{{ response.get('pages', {}).get('next') }}"
|
|
stop_condition: "{{ 'next' not in response.get('pages', {}) }}"
|
|
contacts:
|
|
$ref: "#/definitions/stream_incremental_search"
|
|
$parameters:
|
|
name: "contacts"
|
|
path: "contacts/search"
|
|
page_size: 150
|
|
conversations:
|
|
$ref: "#/definitions/stream_incremental_search"
|
|
$parameters:
|
|
name: "conversations"
|
|
path: "conversations/search"
|
|
data_field: "conversations"
|
|
page_size: 150
|
|
|
|
streams:
|
|
- "#/definitions/admins"
|
|
- "#/definitions/tags"
|
|
- "#/definitions/teams"
|
|
- "#/definitions/segments"
|
|
- "#/definitions/companies"
|
|
- "#/definitions/company_attributes"
|
|
- "#/definitions/contact_attributes"
|
|
- "#/definitions/contacts"
|
|
- "#/definitions/conversations"
|
|
- "#/definitions/conversation_parts"
|
|
- "#/definitions/company_segments"
|
|
|
|
check:
|
|
stream_names:
|
|
- "tags"
|