Source Greenhouse: support incremental syncs (#16338)
* #1386 Source Greenhouse: support incremental syncs - first try * #1386 Source Greenhouse: implement incremental syncs * #1386 source greenhouse: upd changelog * Increased unittest to 90 * Updated link in spec * auto-bump connector version [ci skip] * Updated release stage Co-authored-by: Serhii Lazebnyi <serhii.lazebnyi@globallogic.com> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
This commit is contained in:
@@ -403,11 +403,11 @@
|
||||
- name: Greenhouse
|
||||
sourceDefinitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44
|
||||
dockerRepository: airbyte/source-greenhouse
|
||||
dockerImageTag: 0.2.9
|
||||
dockerImageTag: 0.2.10
|
||||
documentationUrl: https://docs.airbyte.io/integrations/sources/greenhouse
|
||||
icon: greenhouse.svg
|
||||
sourceType: api
|
||||
releaseStage: alpha
|
||||
releaseStage: beta
|
||||
- name: Harness
|
||||
sourceDefinitionId: 6fe89830-d04d-401b-aad6-6552ffa5c4af
|
||||
dockerRepository: farosai/airbyte-harness-source
|
||||
|
||||
@@ -3661,9 +3661,9 @@
|
||||
supportsNormalization: false
|
||||
supportsDBT: false
|
||||
supported_destination_sync_modes: []
|
||||
- dockerImage: "airbyte/source-greenhouse:0.2.9"
|
||||
- dockerImage: "airbyte/source-greenhouse:0.2.10"
|
||||
spec:
|
||||
documentationUrl: "https://docs.airbyte.io/integrations/sources/greenhouse"
|
||||
documentationUrl: "https://docs.airbyte.com/integrations/sources/greenhouse"
|
||||
connectionSpecification:
|
||||
$schema: "http://json-schema.org/draft-07/schema#"
|
||||
title: "Greenhouse Spec"
|
||||
|
||||
@@ -4,13 +4,13 @@ FROM python:3.9-slim
|
||||
RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /airbyte/integration_code
|
||||
COPY source_greenhouse ./source_greenhouse
|
||||
COPY main.py ./
|
||||
COPY setup.py ./
|
||||
RUN pip install .
|
||||
COPY source_greenhouse ./source_greenhouse
|
||||
COPY main.py ./
|
||||
|
||||
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
|
||||
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
|
||||
|
||||
LABEL io.airbyte.version=0.2.9
|
||||
LABEL io.airbyte.version=0.2.10
|
||||
LABEL io.airbyte.name=airbyte/source-greenhouse
|
||||
|
||||
@@ -19,13 +19,14 @@ tests:
|
||||
configured_catalog_path: "integration_tests/configured_catalog.json"
|
||||
expect_records:
|
||||
path: "integration_tests/expected_records.txt"
|
||||
extra_fields: yes
|
||||
exact_order: yes
|
||||
extra_records: no
|
||||
- config_path: "secrets/config.json"
|
||||
configured_catalog_path: "integration_tests/configured_catalog_users_only.json"
|
||||
full_refresh:
|
||||
- config_path: "secrets/config.json"
|
||||
configured_catalog_path: "integration_tests/configured_catalog_const_records.json"
|
||||
configured_catalog_path: "integration_tests/configured_catalog.json"
|
||||
- config_path: "secrets/config_users_only.json"
|
||||
configured_catalog_path: "integration_tests/configured_catalog_users_only.json"
|
||||
incremental:
|
||||
- config_path: "secrets/config.json"
|
||||
configured_catalog_path: "integration_tests/incremental_configured_catalog.json"
|
||||
future_state_path: "integration_tests/abnormal_state.json"
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"candidates": {
|
||||
"updated_at": "2222-01-21T00:00:00.000Z"
|
||||
},
|
||||
"demographics_answers": {
|
||||
"updated_at": "2222-01-21T00:00:00.000Z"
|
||||
},
|
||||
"users": {
|
||||
"updated_at": "2222-01-21T00:00:00.000Z"
|
||||
},
|
||||
"scorecards": {
|
||||
"updated_at": "2222-01-21T00:00:00.000Z"
|
||||
},
|
||||
"offers": {
|
||||
"updated_at": "2222-01-21T00:00:00.000Z"
|
||||
},
|
||||
"job_stages": {
|
||||
"updated_at": "2222-01-21T00:00:00.000Z"
|
||||
},
|
||||
"job_posts": {
|
||||
"updated_at": "2222-01-21T00:00:00.000Z"
|
||||
},
|
||||
"interviews": {
|
||||
"updated_at": "2222-01-21T00:00:00.000Z"
|
||||
},
|
||||
"jobs": {
|
||||
"updated_at": "2222-01-21T00:00:00.000Z"
|
||||
},
|
||||
"applications": {
|
||||
"applied_at": "2222-01-21T00:00:00.000Z"
|
||||
},
|
||||
"jobs_stages": {
|
||||
"4177046003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"4177048003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"4446240003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"4466310003": {"updated_at": "2222-01-01T00:00:00.000Z"}
|
||||
},
|
||||
"applications_demographics_answers": {
|
||||
"19214950003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"19214993003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"19215172003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"19215333003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"44933447003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"44937562003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"47459993003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"48693310003": {"updated_at": "2222-01-01T00:00:00.000Z"}
|
||||
},
|
||||
"applications_interviews": {
|
||||
"19214950003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"19214993003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"19215172003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"19215333003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"44933447003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"44937562003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"47459993003": {"updated_at": "2222-01-01T00:00:00.000Z"},
|
||||
"48693310003": {"updated_at": "2222-01-01T00:00:00.000Z"}
|
||||
}
|
||||
}
|
||||
@@ -4,20 +4,28 @@
|
||||
"stream": {
|
||||
"name": "applications",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["applied_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["applied_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "candidates",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
@@ -25,9 +33,11 @@
|
||||
"name": "close_reasons",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
"source_defined_cursor": false,
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
@@ -35,9 +45,11 @@
|
||||
"name": "degrees",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
"source_defined_cursor": false,
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
@@ -45,59 +57,81 @@
|
||||
"name": "departments",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
"source_defined_cursor": false,
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "job_posts",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "jobs",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "offers",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "scorecards",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "users",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
@@ -105,135 +139,191 @@
|
||||
"name": "custom_fields",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
"source_defined_cursor": false,
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_question_sets",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false,
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_questions",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false,
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_answer_options",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false,
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_answers",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "applications_demographics_answers",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_question_sets_questions",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false,
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_answers_answer_options",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false,
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "interviews",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "applications_interviews",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "sources",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false,
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "rejection_reasons",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false,
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "jobs_openings",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false,
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "job_stages",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "jobs_stages",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,220 +0,0 @@
|
||||
{
|
||||
"streams": [
|
||||
{
|
||||
"stream": {
|
||||
"name": "close_reasons",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "degrees",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "departments",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "job_posts",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "jobs",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "offers",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "scorecards",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "users",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "custom_fields",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "interviews",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "applications_interviews",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "sources",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "rejection_reasons",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "jobs_openings",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "job_stages",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "jobs_stages",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_question_sets",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_questions",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_answer_options",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_answers",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "applications_demographics_answers",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_question_sets_questions",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_answers_answer_options",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -4,10 +4,14 @@
|
||||
"stream": {
|
||||
"name": "users",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"],
|
||||
"source_defined_cursor": false
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_primary_key": [["id"]],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
}
|
||||
]
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,186 @@
|
||||
{
|
||||
"streams": [
|
||||
{
|
||||
"stream": {
|
||||
"name": "applications",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["applied_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["applied_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "candidates",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "job_posts",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "jobs",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "offers",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "scorecards",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "users",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "demographics_answers",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "applications_demographics_answers",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "interviews",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "applications_interviews",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "job_stages",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "jobs_stages",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["updated_at"],
|
||||
"source_defined_primary_key": [["id"]]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"primary_key": [["id"]],
|
||||
"cursor_field": ["updated_at"],
|
||||
"destination_sync_mode": "overwrite"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -16,7 +16,7 @@ setup(
|
||||
author="Airbyte",
|
||||
author_email="contact@airbyte.io",
|
||||
packages=find_packages(),
|
||||
install_requires=["airbyte-cdk~=0.1.79"],
|
||||
install_requires=["airbyte-cdk~=0.1.79", "dataclasses-jsonschema==2.15.1"],
|
||||
package_data={"": ["*.json", "*.yaml", "schemas/*.json"]},
|
||||
extras_require={
|
||||
"tests": TEST_REQUIREMENTS,
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
#
|
||||
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
import datetime
|
||||
from dataclasses import InitVar, dataclass
|
||||
from typing import Any, ClassVar, Iterable, Mapping, MutableMapping, Optional, Union
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer
|
||||
from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState
|
||||
from airbyte_cdk.sources.streams.core import Stream
|
||||
|
||||
|
||||
@dataclass
|
||||
class GreenHouseSlicer(StreamSlicer):
|
||||
options: InitVar[Mapping[str, Any]]
|
||||
cursor_field: str
|
||||
request_cursor_field: str
|
||||
|
||||
START_DATETIME: ClassVar[str] = "1970-01-01T00:00:00.000Z"
|
||||
DATETIME_FORMAT: ClassVar[str] = "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||
|
||||
def __post_init__(self, options: Mapping[str, Any]):
|
||||
self._state = {}
|
||||
|
||||
def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState, *args, **kwargs) -> Iterable[StreamSlice]:
|
||||
yield {self.request_cursor_field: stream_state.get(self.cursor_field, self.START_DATETIME)}
|
||||
|
||||
def _max_dt_str(self, *args: str) -> Optional[str]:
|
||||
new_state_candidates = list(map(lambda x: datetime.datetime.strptime(x, self.DATETIME_FORMAT), filter(None, args)))
|
||||
if not new_state_candidates:
|
||||
return
|
||||
max_dt = max(new_state_candidates)
|
||||
# `.%f` gives us microseconds, we need milliseconds
|
||||
(dt, micro) = max_dt.strftime(self.DATETIME_FORMAT).split(".")
|
||||
return "%s.%03dZ" % (dt, int(micro[:-1:]) / 1000)
|
||||
|
||||
def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None):
|
||||
# stream_state can be passed in as a stream_slice parameter - it's a framework flaw, so we have to workaround it
|
||||
slice_state = stream_slice.get(self.cursor_field)
|
||||
current_state = self._state.get(self.cursor_field)
|
||||
last_cursor = last_record and last_record[self.cursor_field]
|
||||
max_dt = self._max_dt_str(slice_state, current_state, last_cursor)
|
||||
if not max_dt:
|
||||
return
|
||||
self._state[self.cursor_field] = max_dt
|
||||
|
||||
def get_stream_state(self) -> StreamState:
|
||||
return self._state
|
||||
|
||||
def get_request_params(
|
||||
self,
|
||||
*,
|
||||
stream_state: Optional[StreamState] = None,
|
||||
stream_slice: Optional[StreamSlice] = None,
|
||||
next_page_token: Optional[Mapping[str, Any]] = None,
|
||||
) -> MutableMapping[str, Any]:
|
||||
return stream_slice or {}
|
||||
|
||||
def get_request_headers(self, *args, **kwargs) -> Mapping[str, Any]:
|
||||
return {}
|
||||
|
||||
def get_request_body_data(self, *args, **kwargs) -> Optional[Union[Mapping, str]]:
|
||||
return {}
|
||||
|
||||
def get_request_body_json(self, *args, **kwargs) -> Optional[Mapping]:
|
||||
return {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class GreenHouseSubstreamSlicer(GreenHouseSlicer):
|
||||
parent_stream: Stream
|
||||
stream_slice_field: str
|
||||
parent_key: str
|
||||
|
||||
def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Iterable[StreamSlice]:
|
||||
for parent_stream_slice in self.parent_stream.stream_slices(sync_mode=sync_mode, cursor_field=None, stream_state=stream_state):
|
||||
for parent_record in self.parent_stream.read_records(
|
||||
sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice=parent_stream_slice, stream_state=None
|
||||
):
|
||||
parent_state_value = parent_record.get(self.parent_key)
|
||||
yield {
|
||||
self.stream_slice_field: parent_state_value,
|
||||
self.request_cursor_field: stream_state.get(str(parent_state_value), {}).get(self.cursor_field, self.START_DATETIME),
|
||||
}
|
||||
|
||||
def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None):
|
||||
if last_record:
|
||||
# stream_slice is really a stream slice
|
||||
substream_id = str(stream_slice[self.stream_slice_field])
|
||||
current_state = self._state.get(substream_id, {}).get(self.cursor_field)
|
||||
last_state = last_record[self.cursor_field]
|
||||
max_dt = self._max_dt_str(last_state, current_state)
|
||||
self._state[substream_id] = {self.cursor_field: max_dt}
|
||||
return
|
||||
# stream_slice here may be a stream slice or a state
|
||||
if self.stream_slice_field in stream_slice:
|
||||
return
|
||||
substream_ids = map(lambda x: str(x), set(stream_slice.keys()) | set(self._state.keys()))
|
||||
for id_ in substream_ids:
|
||||
self._state[id_] = {
|
||||
self.cursor_field: self._max_dt_str(
|
||||
stream_slice.get(id_, {}).get(self.cursor_field), self._state.get(id_, {}).get(self.cursor_field)
|
||||
)
|
||||
}
|
||||
|
||||
def get_request_params(
|
||||
self,
|
||||
*,
|
||||
stream_state: Optional[StreamState] = None,
|
||||
stream_slice: Optional[StreamSlice] = None,
|
||||
next_page_token: Optional[Mapping[str, Any]] = None,
|
||||
) -> MutableMapping[str, Any]:
|
||||
# ignore other fields in a slice
|
||||
return {self.request_cursor_field: stream_slice[self.request_cursor_field]}
|
||||
@@ -45,18 +45,34 @@ definitions:
|
||||
$ref: "*ref(definitions.retriever)"
|
||||
requester:
|
||||
$ref: "*ref(definitions.requester)"
|
||||
applications_stream:
|
||||
base_incremental_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
stream_cursor_field: "updated_at"
|
||||
retriever:
|
||||
$ref: "*ref(definitions.retriever)"
|
||||
requester: "*ref(definitions.requester)"
|
||||
stream_slicer:
|
||||
request_cursor_field: "updated_after"
|
||||
cursor_field: "updated_at"
|
||||
class_name: source_greenhouse.components.GreenHouseSlicer
|
||||
applications_stream:
|
||||
$ref: "*ref(definitions.base_incremental_stream)"
|
||||
$options:
|
||||
name: "applications"
|
||||
path: "applications"
|
||||
primary_key: "id"
|
||||
stream_cursor_field: "applied_at"
|
||||
retriever:
|
||||
$ref: "*ref(definitions.retriever)"
|
||||
requester: "*ref(definitions.requester)"
|
||||
stream_slicer:
|
||||
request_cursor_field: "created_after"
|
||||
cursor_field: "applied_at"
|
||||
class_name: source_greenhouse.components.GreenHouseSlicer
|
||||
candidates_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$ref: "*ref(definitions.base_incremental_stream)"
|
||||
$options:
|
||||
name: "candidates"
|
||||
path: "candidates"
|
||||
primary_key: "id"
|
||||
close_reasons_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$options:
|
||||
@@ -74,7 +90,7 @@ definitions:
|
||||
name: "departments"
|
||||
path: "departments"
|
||||
jobs_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$ref: "*ref(definitions.base_incremental_stream)"
|
||||
$options:
|
||||
name: "jobs"
|
||||
path: "jobs"
|
||||
@@ -96,39 +112,39 @@ definitions:
|
||||
parent_key: "id"
|
||||
stream_slice_field: "parent_id"
|
||||
applications_demographics_answers_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$options:
|
||||
name: "applications_demographics_answers"
|
||||
primary_key: "id"
|
||||
schema_loader:
|
||||
$ref: "*ref(definitions.schema_loader)"
|
||||
stream_cursor_field: "updated_at"
|
||||
retriever:
|
||||
$ref: "*ref(definitions.retriever)"
|
||||
requester:
|
||||
$ref: "*ref(definitions.requester)"
|
||||
path: "applications/{{ stream_slice.parent_id }}/demographics/answers"
|
||||
stream_slicer:
|
||||
type: SubstreamSlicer
|
||||
parent_stream_configs:
|
||||
- stream: "*ref(definitions.applications_stream)"
|
||||
parent_key: "id"
|
||||
stream_slice_field: "parent_id"
|
||||
class_name: source_greenhouse.components.GreenHouseSubstreamSlicer
|
||||
parent_stream: "*ref(definitions.applications_stream)"
|
||||
request_cursor_field: "updated_after"
|
||||
stream_slice_field: "parent_id"
|
||||
cursor_field: "updated_at"
|
||||
parent_key: "id"
|
||||
applications_interviews_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$options:
|
||||
name: "applications_interviews"
|
||||
primary_key: "id"
|
||||
schema_loader:
|
||||
$ref: "*ref(definitions.schema_loader)"
|
||||
stream_cursor_field: "updated_at"
|
||||
retriever:
|
||||
$ref: "*ref(definitions.retriever)"
|
||||
requester:
|
||||
$ref: "*ref(definitions.requester)"
|
||||
path: "applications/{{ stream_slice.parent_id }}/scheduled_interviews"
|
||||
stream_slicer:
|
||||
type: SubstreamSlicer
|
||||
parent_stream_configs:
|
||||
- stream: "*ref(definitions.applications_stream)"
|
||||
parent_key: "id"
|
||||
stream_slice_field: "parent_id"
|
||||
class_name: source_greenhouse.components.GreenHouseSubstreamSlicer
|
||||
parent_stream: "*ref(definitions.applications_stream)"
|
||||
request_cursor_field: "updated_after"
|
||||
stream_slice_field: "parent_id"
|
||||
cursor_field: "updated_at"
|
||||
parent_key: "id"
|
||||
custom_fields_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$options:
|
||||
@@ -179,39 +195,40 @@ definitions:
|
||||
parent_key: "id"
|
||||
stream_slice_field: "parent_id"
|
||||
interviews_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$ref: "*ref(definitions.base_incremental_stream)"
|
||||
$options:
|
||||
name: "interviews"
|
||||
path: "scheduled_interviews"
|
||||
job_posts_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$ref: "*ref(definitions.base_incremental_stream)"
|
||||
$options:
|
||||
name: "job_posts"
|
||||
path: "job_posts"
|
||||
job_stages_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$ref: "*ref(definitions.base_incremental_stream)"
|
||||
$options:
|
||||
name: "job_stages"
|
||||
path: "job_stages"
|
||||
jobs_stages_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$options:
|
||||
name: "jobs_stages"
|
||||
primary_key: "id"
|
||||
schema_loader:
|
||||
$ref: "*ref(definitions.schema_loader)"
|
||||
retriever:
|
||||
$ref: "*ref(definitions.retriever)"
|
||||
requester:
|
||||
$ref: "*ref(definitions.requester)"
|
||||
path: "jobs/{{ stream_slice.parent_id }}/stages"
|
||||
stream_slicer:
|
||||
type: SubstreamSlicer
|
||||
parent_stream_configs:
|
||||
- stream: "*ref(definitions.jobs_stream)"
|
||||
parent_key: "id"
|
||||
stream_slice_field: "parent_id"
|
||||
path: "jobs/{{ stream_slice.parent_id }}/stages"
|
||||
stream_cursor_field: "updated_at"
|
||||
retriever:
|
||||
$ref: "*ref(definitions.retriever)"
|
||||
requester:
|
||||
$ref: "*ref(definitions.requester)"
|
||||
path: "jobs/{{ stream_slice.parent_id }}/stages"
|
||||
stream_slicer:
|
||||
class_name: source_greenhouse.components.GreenHouseSubstreamSlicer
|
||||
parent_stream: "*ref(definitions.jobs_stream)"
|
||||
request_cursor_field: "updated_after"
|
||||
stream_slice_field: "parent_id"
|
||||
cursor_field: "updated_at"
|
||||
parent_key: "id"
|
||||
offers_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$ref: "*ref(definitions.base_incremental_stream)"
|
||||
$options:
|
||||
name: "offers"
|
||||
path: "offers"
|
||||
@@ -221,7 +238,7 @@ definitions:
|
||||
name: "rejection_reasons"
|
||||
path: "rejection_reasons"
|
||||
scorecards_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$ref: "*ref(definitions.base_incremental_stream)"
|
||||
$options:
|
||||
name: "scorecards"
|
||||
path: "scorecards"
|
||||
@@ -231,12 +248,12 @@ definitions:
|
||||
name: "sources"
|
||||
path: "sources"
|
||||
users_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$ref: "*ref(definitions.base_incremental_stream)"
|
||||
$options:
|
||||
name: "users"
|
||||
path: "users"
|
||||
demographics_answers_stream:
|
||||
$ref: "*ref(definitions.base_stream)"
|
||||
$ref: "*ref(definitions.base_incremental_stream)"
|
||||
$options:
|
||||
name: "demographics_answers"
|
||||
path: "demographics/answers"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"documentationUrl": "https://docs.airbyte.io/integrations/sources/greenhouse",
|
||||
"documentationUrl": "https://docs.airbyte.com/integrations/sources/greenhouse",
|
||||
"connectionSpecification": {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Greenhouse Spec",
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
#
|
||||
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from source_greenhouse.components import GreenHouseSlicer, GreenHouseSubstreamSlicer
|
||||
|
||||
|
||||
def test_slicer():
|
||||
date_time = "2022-09-05T10:10:10.000000Z"
|
||||
date_time_dict = {date_time: date_time}
|
||||
slicer = GreenHouseSlicer(cursor_field=date_time, options=None, request_cursor_field=None)
|
||||
slicer.update_cursor(stream_slice=date_time_dict, last_record=date_time_dict)
|
||||
assert slicer.get_stream_state() == {date_time: "2022-09-05T10:10:10.000Z"}
|
||||
assert slicer.get_request_headers() == {}
|
||||
assert slicer.get_request_body_data() == {}
|
||||
assert slicer.get_request_body_json() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"last_record, expected, records",
|
||||
[
|
||||
(
|
||||
{"2022-09-05T10:10:10.000000Z": "2022-09-05T10:10:10.000000Z"},
|
||||
{"parent_key": {"2022-09-05T10:10:10.000000Z": "2022-09-05T10:10:10.000Z"}},
|
||||
[{"parent_key": "parent_key"}],
|
||||
),
|
||||
(None, {}, []),
|
||||
],
|
||||
)
|
||||
def test_sub_slicer(last_record, expected, records):
|
||||
date_time = "2022-09-05T10:10:10.000000Z"
|
||||
parent_slicer = GreenHouseSlicer(cursor_field=date_time, options=None, request_cursor_field=None)
|
||||
GreenHouseSlicer.read_records = MagicMock(return_value=records)
|
||||
slicer = GreenHouseSubstreamSlicer(
|
||||
cursor_field=date_time,
|
||||
options=None,
|
||||
request_cursor_field=None,
|
||||
parent_stream=parent_slicer,
|
||||
stream_slice_field=date_time,
|
||||
parent_key="parent_key",
|
||||
)
|
||||
stream_slice = next(slicer.stream_slices(SyncMode, {})) if records else {}
|
||||
slicer.update_cursor(stream_slice=stream_slice, last_record=last_record)
|
||||
assert slicer.get_stream_state() == expected
|
||||
@@ -64,11 +64,12 @@ Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see
|
||||
|
||||
## Changelog
|
||||
|
||||
| Version | Date | Pull Request | Subject |
|
||||
|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------|
|
||||
| 0.2.9 | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime |
|
||||
| 0.2.8 | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework |
|
||||
| 0.2.7 | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users |
|
||||
| 0.2.6 | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607) | Implement demographics streams support. Update SAT for demographics streams |
|
||||
| 0.2.5 | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377) | Refactor the connector to use CDK. Implement additional stream support |
|
||||
| Version | Date | Pull Request | Subject |
|
||||
|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------|
|
||||
| 0.2.10 | 2022-09-05 | [16338](https://github.com/airbytehq/airbyte/pull/16338) | Implement incremental syncs & fix SATs |
|
||||
| 0.2.9 | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime |
|
||||
| 0.2.8 | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework |
|
||||
| 0.2.7 | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users |
|
||||
| 0.2.6 | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607) | Implement demographics streams support. Update SAT for demographics streams |
|
||||
| 0.2.5 | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377) | Refactor the connector to use CDK. Implement additional stream support |
|
||||
| 0.2.4 | 2021-09-15 | [6238](https://github.com/airbytehq/airbyte/pull/6238) | Add identification of accessible streams for API keys with limited permissions |
|
||||
|
||||
Reference in New Issue
Block a user