903d34e5f1
* enforce manifest version correctness against the CDK package being used * parse versions into parts for better comparisons and error checking * fix pr feedback and derp forgot to actually add the commit with the low-code manifests updated to the beta version * pr feedback and fix new tests since last rebase
143 lines
4.1 KiB
YAML
143 lines
4.1 KiB
YAML
version: "0.29.0"
|
|
|
|
definitions:
|
|
selector:
|
|
type: RecordSelector
|
|
extractor:
|
|
type: DpathExtractor
|
|
field_path: ["{{ parameters['data_field'] }}"]
|
|
requester:
|
|
type: HttpRequester
|
|
http_method: "GET"
|
|
authenticator:
|
|
type: BearerAuthenticator
|
|
api_token: "{{ config['api_token'] }}"
|
|
retriever:
|
|
type: SimpleRetriever
|
|
$parameters:
|
|
url_base: "https://api.zenloop.com/v1/"
|
|
record_selector:
|
|
$ref: "#/definitions/selector"
|
|
paginator:
|
|
type: DefaultPaginator
|
|
pagination_strategy:
|
|
type: PageIncrement
|
|
page_size: 50
|
|
start_from_page: 1
|
|
page_size_option:
|
|
field_name: "per_page"
|
|
inject_into: "request_parameter"
|
|
page_token_option:
|
|
type: RequestPath
|
|
base_stream:
|
|
type: DeclarativeStream
|
|
retriever:
|
|
$ref: "#/definitions/retriever"
|
|
requester:
|
|
$ref: "#/definitions/requester"
|
|
incremental_base_stream:
|
|
incremental_sync:
|
|
$ref: "#/definitions/incremental_sync"
|
|
$ref: "#/definitions/base_stream"
|
|
retriever:
|
|
$ref: "#/definitions/retriever"
|
|
requester:
|
|
$ref: "#/definitions/requester"
|
|
request_parameters:
|
|
order_type: "desc"
|
|
order_by: "inserted_at"
|
|
date_shortcut: "custom"
|
|
surveys:
|
|
$ref: "#/definitions/base_stream"
|
|
$parameters:
|
|
name: "surveys"
|
|
path: "surveys"
|
|
data_field: "surveys"
|
|
surveys_slicer:
|
|
class_name: source_zenloop.components.ZenloopPartitionRouter
|
|
$parameters:
|
|
config_parent_field: "survey_id"
|
|
parent_stream_configs:
|
|
- stream: "#/definitions/surveys"
|
|
parent_key: public_hash_id
|
|
partition_field: id
|
|
survey_groups:
|
|
$ref: "#/definitions/base_stream"
|
|
$parameters:
|
|
name: "survey_groups"
|
|
path: "survey_groups"
|
|
data_field: "survey_groups"
|
|
survey_groups_slicer:
|
|
class_name: source_zenloop.components.ZenloopPartitionRouter
|
|
$parameters:
|
|
config_parent_field: "survey_group_id"
|
|
parent_stream_configs:
|
|
- stream: "#/definitions/survey_groups"
|
|
parent_key: public_hash_id
|
|
partition_field: id
|
|
incremental_sync:
|
|
type: DatetimeBasedCursor
|
|
cursor_field: "inserted_at"
|
|
datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ"
|
|
cursor_granularity: "PT0.000001S"
|
|
start_datetime:
|
|
datetime: "{{ config['date_from'] }}"
|
|
datetime_format: "%Y-%m-%d"
|
|
end_datetime:
|
|
datetime: "{{ today_utc() }}"
|
|
datetime_format: "%Y-%m-%d"
|
|
step: "P1M"
|
|
end_time_option:
|
|
field_name: "date_to"
|
|
inject_into: "request_parameter"
|
|
start_time_option:
|
|
field_name: "date_from"
|
|
inject_into: "request_parameter"
|
|
properties:
|
|
$ref: "#/definitions/base_stream"
|
|
$parameters:
|
|
name: "properties"
|
|
data_field: "properties"
|
|
retriever:
|
|
$ref: "#/definitions/retriever"
|
|
requester:
|
|
$ref: "#/definitions/requester"
|
|
path: "{{ 'surveys/' + config['survey_id'] + '/properties' if config['survey_id'] else 'surveys/' + stream_slice.id + '/properties' }}"
|
|
partition_router:
|
|
$ref: "#/definitions/surveys_slicer"
|
|
answers:
|
|
$ref: "#/definitions/incremental_base_stream"
|
|
$parameters:
|
|
name: "answers"
|
|
data_field: "answers"
|
|
retriever:
|
|
$ref: "#/definitions/retriever"
|
|
requester:
|
|
$ref: "#/definitions/incremental_base_stream/retriever/requester"
|
|
path: "{{ 'surveys/' + stream_slice.id + '/answers' }}"
|
|
partition_router:
|
|
$ref: "#/definitions/surveys_slicer"
|
|
answers_survey_group:
|
|
$ref: "#/definitions/incremental_base_stream"
|
|
$parameters:
|
|
name: "answers_survey_group"
|
|
data_field: "answers"
|
|
retriever:
|
|
$ref: "#/definitions/retriever"
|
|
requester:
|
|
$ref: "#/definitions/incremental_base_stream/retriever/requester"
|
|
path: "{{ 'survey_groups/' + stream_slice.id + '/answers' }}"
|
|
partition_router:
|
|
$ref: "#/definitions/survey_groups_slicer"
|
|
|
|
streams:
|
|
- "#/definitions/surveys"
|
|
- "#/definitions/survey_groups"
|
|
- "#/definitions/properties"
|
|
- "#/definitions/answers"
|
|
- "#/definitions/answers_survey_group"
|
|
|
|
check:
|
|
type: CheckStream
|
|
stream_names: ["surveys"]
|