1
0
mirror of synced 2025-12-25 02:09:19 -05:00

Source Greenhouse: support incremental syncs (#16338)

* #1386 Source Greenhouse: support incremental syncs - first try

* #1386 Source Greenhouse: implement incremental syncs

* #1386 source greenhouse: upd changelog

* Increased unittest to 90

* Updated link in spec

* auto-bump connector version [ci skip]

* Updated release stage

Co-authored-by: Serhii Lazebnyi <serhii.lazebnyi@globallogic.com>
Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
This commit is contained in:
Denys Davydov
2022-09-06 00:13:25 +03:00
committed by GitHub
parent 498d70089f
commit 8b5362ef11
16 changed files with 645 additions and 346 deletions

View File

@@ -403,11 +403,11 @@
- name: Greenhouse
sourceDefinitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44
dockerRepository: airbyte/source-greenhouse
dockerImageTag: 0.2.9
dockerImageTag: 0.2.10
documentationUrl: https://docs.airbyte.io/integrations/sources/greenhouse
icon: greenhouse.svg
sourceType: api
releaseStage: alpha
releaseStage: beta
- name: Harness
sourceDefinitionId: 6fe89830-d04d-401b-aad6-6552ffa5c4af
dockerRepository: farosai/airbyte-harness-source

View File

@@ -3661,9 +3661,9 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-greenhouse:0.2.9"
- dockerImage: "airbyte/source-greenhouse:0.2.10"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/greenhouse"
documentationUrl: "https://docs.airbyte.com/integrations/sources/greenhouse"
connectionSpecification:
$schema: "http://json-schema.org/draft-07/schema#"
title: "Greenhouse Spec"

View File

@@ -4,13 +4,13 @@ FROM python:3.9-slim
RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
WORKDIR /airbyte/integration_code
COPY source_greenhouse ./source_greenhouse
COPY main.py ./
COPY setup.py ./
RUN pip install .
COPY source_greenhouse ./source_greenhouse
COPY main.py ./
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.2.9
LABEL io.airbyte.version=0.2.10
LABEL io.airbyte.name=airbyte/source-greenhouse

View File

@@ -19,13 +19,14 @@ tests:
configured_catalog_path: "integration_tests/configured_catalog.json"
expect_records:
path: "integration_tests/expected_records.txt"
extra_fields: yes
exact_order: yes
extra_records: no
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog_users_only.json"
full_refresh:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog_const_records.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
- config_path: "secrets/config_users_only.json"
configured_catalog_path: "integration_tests/configured_catalog_users_only.json"
incremental:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/incremental_configured_catalog.json"
future_state_path: "integration_tests/abnormal_state.json"

View File

@@ -0,0 +1,58 @@
{
"candidates": {
"updated_at": "2222-01-21T00:00:00.000Z"
},
"demographics_answers": {
"updated_at": "2222-01-21T00:00:00.000Z"
},
"users": {
"updated_at": "2222-01-21T00:00:00.000Z"
},
"scorecards": {
"updated_at": "2222-01-21T00:00:00.000Z"
},
"offers": {
"updated_at": "2222-01-21T00:00:00.000Z"
},
"job_stages": {
"updated_at": "2222-01-21T00:00:00.000Z"
},
"job_posts": {
"updated_at": "2222-01-21T00:00:00.000Z"
},
"interviews": {
"updated_at": "2222-01-21T00:00:00.000Z"
},
"jobs": {
"updated_at": "2222-01-21T00:00:00.000Z"
},
"applications": {
"applied_at": "2222-01-21T00:00:00.000Z"
},
"jobs_stages": {
"4177046003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"4177048003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"4446240003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"4466310003": {"updated_at": "2222-01-01T00:00:00.000Z"}
},
"applications_demographics_answers": {
"19214950003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"19214993003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"19215172003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"19215333003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"44933447003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"44937562003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"47459993003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"48693310003": {"updated_at": "2222-01-01T00:00:00.000Z"}
},
"applications_interviews": {
"19214950003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"19214993003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"19215172003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"19215333003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"44933447003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"44937562003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"47459993003": {"updated_at": "2222-01-01T00:00:00.000Z"},
"48693310003": {"updated_at": "2222-01-01T00:00:00.000Z"}
}
}

View File

@@ -4,20 +4,28 @@
"stream": {
"name": "applications",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["applied_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["applied_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "candidates",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
@@ -25,9 +33,11 @@
"name": "close_reasons",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
"source_defined_cursor": false,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"destination_sync_mode": "overwrite"
},
{
@@ -35,9 +45,11 @@
"name": "degrees",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
"source_defined_cursor": false,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"destination_sync_mode": "overwrite"
},
{
@@ -45,59 +57,81 @@
"name": "departments",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
"source_defined_cursor": false,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "job_posts",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "jobs",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "offers",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "scorecards",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "users",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
@@ -105,135 +139,191 @@
"name": "custom_fields",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
"source_defined_cursor": false,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_question_sets",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_questions",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_answer_options",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_answers",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "applications_demographics_answers",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_question_sets_questions",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_answers_answer_options",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "interviews",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "applications_interviews",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "sources",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "rejection_reasons",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "jobs_openings",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "job_stages",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "jobs_stages",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
}
]

View File

@@ -1,220 +0,0 @@
{
"streams": [
{
"stream": {
"name": "close_reasons",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "degrees",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "departments",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "job_posts",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "jobs",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "offers",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "scorecards",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "users",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "custom_fields",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "interviews",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "applications_interviews",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "sources",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "rejection_reasons",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "jobs_openings",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "job_stages",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "jobs_stages",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_question_sets",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_questions",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_answer_options",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_answers",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "applications_demographics_answers",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_question_sets_questions",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_answers_answer_options",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
}
]
}

View File

@@ -4,10 +4,14 @@
"stream": {
"name": "users",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_primary_key": [["id"]],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"]
},
"sync_mode": "full_refresh",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
}
]

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,186 @@
{
"streams": [
{
"stream": {
"name": "applications",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["applied_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["applied_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "candidates",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "job_posts",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "jobs",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "offers",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "scorecards",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "users",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "demographics_answers",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "applications_demographics_answers",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "interviews",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "applications_interviews",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "job_stages",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "jobs_stages",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"primary_key": [["id"]],
"cursor_field": ["updated_at"],
"destination_sync_mode": "overwrite"
}
]
}

View File

@@ -16,7 +16,7 @@ setup(
author="Airbyte",
author_email="contact@airbyte.io",
packages=find_packages(),
install_requires=["airbyte-cdk~=0.1.79"],
install_requires=["airbyte-cdk~=0.1.79", "dataclasses-jsonschema==2.15.1"],
package_data={"": ["*.json", "*.yaml", "schemas/*.json"]},
extras_require={
"tests": TEST_REQUIREMENTS,

View File

@@ -0,0 +1,116 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
import datetime
from dataclasses import InitVar, dataclass
from typing import Any, ClassVar, Iterable, Mapping, MutableMapping, Optional, Union
from airbyte_cdk.models import SyncMode
from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer
from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState
from airbyte_cdk.sources.streams.core import Stream
@dataclass
class GreenHouseSlicer(StreamSlicer):
options: InitVar[Mapping[str, Any]]
cursor_field: str
request_cursor_field: str
START_DATETIME: ClassVar[str] = "1970-01-01T00:00:00.000Z"
DATETIME_FORMAT: ClassVar[str] = "%Y-%m-%dT%H:%M:%S.%fZ"
def __post_init__(self, options: Mapping[str, Any]):
self._state = {}
def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState, *args, **kwargs) -> Iterable[StreamSlice]:
yield {self.request_cursor_field: stream_state.get(self.cursor_field, self.START_DATETIME)}
def _max_dt_str(self, *args: str) -> Optional[str]:
new_state_candidates = list(map(lambda x: datetime.datetime.strptime(x, self.DATETIME_FORMAT), filter(None, args)))
if not new_state_candidates:
return
max_dt = max(new_state_candidates)
# `.%f` gives us microseconds, we need milliseconds
(dt, micro) = max_dt.strftime(self.DATETIME_FORMAT).split(".")
return "%s.%03dZ" % (dt, int(micro[:-1:]) / 1000)
def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None):
# stream_state can be passed in as a stream_slice parameter - it's a framework flaw, so we have to workaround it
slice_state = stream_slice.get(self.cursor_field)
current_state = self._state.get(self.cursor_field)
last_cursor = last_record and last_record[self.cursor_field]
max_dt = self._max_dt_str(slice_state, current_state, last_cursor)
if not max_dt:
return
self._state[self.cursor_field] = max_dt
def get_stream_state(self) -> StreamState:
return self._state
def get_request_params(
self,
*,
stream_state: Optional[StreamState] = None,
stream_slice: Optional[StreamSlice] = None,
next_page_token: Optional[Mapping[str, Any]] = None,
) -> MutableMapping[str, Any]:
return stream_slice or {}
def get_request_headers(self, *args, **kwargs) -> Mapping[str, Any]:
return {}
def get_request_body_data(self, *args, **kwargs) -> Optional[Union[Mapping, str]]:
return {}
def get_request_body_json(self, *args, **kwargs) -> Optional[Mapping]:
return {}
@dataclass
class GreenHouseSubstreamSlicer(GreenHouseSlicer):
parent_stream: Stream
stream_slice_field: str
parent_key: str
def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Iterable[StreamSlice]:
for parent_stream_slice in self.parent_stream.stream_slices(sync_mode=sync_mode, cursor_field=None, stream_state=stream_state):
for parent_record in self.parent_stream.read_records(
sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice=parent_stream_slice, stream_state=None
):
parent_state_value = parent_record.get(self.parent_key)
yield {
self.stream_slice_field: parent_state_value,
self.request_cursor_field: stream_state.get(str(parent_state_value), {}).get(self.cursor_field, self.START_DATETIME),
}
def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None):
if last_record:
# stream_slice is really a stream slice
substream_id = str(stream_slice[self.stream_slice_field])
current_state = self._state.get(substream_id, {}).get(self.cursor_field)
last_state = last_record[self.cursor_field]
max_dt = self._max_dt_str(last_state, current_state)
self._state[substream_id] = {self.cursor_field: max_dt}
return
# stream_slice here may be a stream slice or a state
if self.stream_slice_field in stream_slice:
return
substream_ids = map(lambda x: str(x), set(stream_slice.keys()) | set(self._state.keys()))
for id_ in substream_ids:
self._state[id_] = {
self.cursor_field: self._max_dt_str(
stream_slice.get(id_, {}).get(self.cursor_field), self._state.get(id_, {}).get(self.cursor_field)
)
}
def get_request_params(
self,
*,
stream_state: Optional[StreamState] = None,
stream_slice: Optional[StreamSlice] = None,
next_page_token: Optional[Mapping[str, Any]] = None,
) -> MutableMapping[str, Any]:
# ignore other fields in a slice
return {self.request_cursor_field: stream_slice[self.request_cursor_field]}

View File

@@ -45,18 +45,34 @@ definitions:
$ref: "*ref(definitions.retriever)"
requester:
$ref: "*ref(definitions.requester)"
applications_stream:
base_incremental_stream:
$ref: "*ref(definitions.base_stream)"
stream_cursor_field: "updated_at"
retriever:
$ref: "*ref(definitions.retriever)"
requester: "*ref(definitions.requester)"
stream_slicer:
request_cursor_field: "updated_after"
cursor_field: "updated_at"
class_name: source_greenhouse.components.GreenHouseSlicer
applications_stream:
$ref: "*ref(definitions.base_incremental_stream)"
$options:
name: "applications"
path: "applications"
primary_key: "id"
stream_cursor_field: "applied_at"
retriever:
$ref: "*ref(definitions.retriever)"
requester: "*ref(definitions.requester)"
stream_slicer:
request_cursor_field: "created_after"
cursor_field: "applied_at"
class_name: source_greenhouse.components.GreenHouseSlicer
candidates_stream:
$ref: "*ref(definitions.base_stream)"
$ref: "*ref(definitions.base_incremental_stream)"
$options:
name: "candidates"
path: "candidates"
primary_key: "id"
close_reasons_stream:
$ref: "*ref(definitions.base_stream)"
$options:
@@ -74,7 +90,7 @@ definitions:
name: "departments"
path: "departments"
jobs_stream:
$ref: "*ref(definitions.base_stream)"
$ref: "*ref(definitions.base_incremental_stream)"
$options:
name: "jobs"
path: "jobs"
@@ -96,39 +112,39 @@ definitions:
parent_key: "id"
stream_slice_field: "parent_id"
applications_demographics_answers_stream:
$ref: "*ref(definitions.base_stream)"
$options:
name: "applications_demographics_answers"
primary_key: "id"
schema_loader:
$ref: "*ref(definitions.schema_loader)"
stream_cursor_field: "updated_at"
retriever:
$ref: "*ref(definitions.retriever)"
requester:
$ref: "*ref(definitions.requester)"
path: "applications/{{ stream_slice.parent_id }}/demographics/answers"
stream_slicer:
type: SubstreamSlicer
parent_stream_configs:
- stream: "*ref(definitions.applications_stream)"
parent_key: "id"
stream_slice_field: "parent_id"
class_name: source_greenhouse.components.GreenHouseSubstreamSlicer
parent_stream: "*ref(definitions.applications_stream)"
request_cursor_field: "updated_after"
stream_slice_field: "parent_id"
cursor_field: "updated_at"
parent_key: "id"
applications_interviews_stream:
$ref: "*ref(definitions.base_stream)"
$options:
name: "applications_interviews"
primary_key: "id"
schema_loader:
$ref: "*ref(definitions.schema_loader)"
stream_cursor_field: "updated_at"
retriever:
$ref: "*ref(definitions.retriever)"
requester:
$ref: "*ref(definitions.requester)"
path: "applications/{{ stream_slice.parent_id }}/scheduled_interviews"
stream_slicer:
type: SubstreamSlicer
parent_stream_configs:
- stream: "*ref(definitions.applications_stream)"
parent_key: "id"
stream_slice_field: "parent_id"
class_name: source_greenhouse.components.GreenHouseSubstreamSlicer
parent_stream: "*ref(definitions.applications_stream)"
request_cursor_field: "updated_after"
stream_slice_field: "parent_id"
cursor_field: "updated_at"
parent_key: "id"
custom_fields_stream:
$ref: "*ref(definitions.base_stream)"
$options:
@@ -179,39 +195,40 @@ definitions:
parent_key: "id"
stream_slice_field: "parent_id"
interviews_stream:
$ref: "*ref(definitions.base_stream)"
$ref: "*ref(definitions.base_incremental_stream)"
$options:
name: "interviews"
path: "scheduled_interviews"
job_posts_stream:
$ref: "*ref(definitions.base_stream)"
$ref: "*ref(definitions.base_incremental_stream)"
$options:
name: "job_posts"
path: "job_posts"
job_stages_stream:
$ref: "*ref(definitions.base_stream)"
$ref: "*ref(definitions.base_incremental_stream)"
$options:
name: "job_stages"
path: "job_stages"
jobs_stages_stream:
$ref: "*ref(definitions.base_stream)"
$options:
name: "jobs_stages"
primary_key: "id"
schema_loader:
$ref: "*ref(definitions.schema_loader)"
retriever:
$ref: "*ref(definitions.retriever)"
requester:
$ref: "*ref(definitions.requester)"
path: "jobs/{{ stream_slice.parent_id }}/stages"
stream_slicer:
type: SubstreamSlicer
parent_stream_configs:
- stream: "*ref(definitions.jobs_stream)"
parent_key: "id"
stream_slice_field: "parent_id"
path: "jobs/{{ stream_slice.parent_id }}/stages"
stream_cursor_field: "updated_at"
retriever:
$ref: "*ref(definitions.retriever)"
requester:
$ref: "*ref(definitions.requester)"
path: "jobs/{{ stream_slice.parent_id }}/stages"
stream_slicer:
class_name: source_greenhouse.components.GreenHouseSubstreamSlicer
parent_stream: "*ref(definitions.jobs_stream)"
request_cursor_field: "updated_after"
stream_slice_field: "parent_id"
cursor_field: "updated_at"
parent_key: "id"
offers_stream:
$ref: "*ref(definitions.base_stream)"
$ref: "*ref(definitions.base_incremental_stream)"
$options:
name: "offers"
path: "offers"
@@ -221,7 +238,7 @@ definitions:
name: "rejection_reasons"
path: "rejection_reasons"
scorecards_stream:
$ref: "*ref(definitions.base_stream)"
$ref: "*ref(definitions.base_incremental_stream)"
$options:
name: "scorecards"
path: "scorecards"
@@ -231,12 +248,12 @@ definitions:
name: "sources"
path: "sources"
users_stream:
$ref: "*ref(definitions.base_stream)"
$ref: "*ref(definitions.base_incremental_stream)"
$options:
name: "users"
path: "users"
demographics_answers_stream:
$ref: "*ref(definitions.base_stream)"
$ref: "*ref(definitions.base_incremental_stream)"
$options:
name: "demographics_answers"
path: "demographics/answers"

View File

@@ -1,5 +1,5 @@
{
"documentationUrl": "https://docs.airbyte.io/integrations/sources/greenhouse",
"documentationUrl": "https://docs.airbyte.com/integrations/sources/greenhouse",
"connectionSpecification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Greenhouse Spec",

View File

@@ -0,0 +1,49 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
from unittest.mock import MagicMock
import pytest
from airbyte_cdk.models import SyncMode
from source_greenhouse.components import GreenHouseSlicer, GreenHouseSubstreamSlicer
def test_slicer():
date_time = "2022-09-05T10:10:10.000000Z"
date_time_dict = {date_time: date_time}
slicer = GreenHouseSlicer(cursor_field=date_time, options=None, request_cursor_field=None)
slicer.update_cursor(stream_slice=date_time_dict, last_record=date_time_dict)
assert slicer.get_stream_state() == {date_time: "2022-09-05T10:10:10.000Z"}
assert slicer.get_request_headers() == {}
assert slicer.get_request_body_data() == {}
assert slicer.get_request_body_json() == {}
@pytest.mark.parametrize(
"last_record, expected, records",
[
(
{"2022-09-05T10:10:10.000000Z": "2022-09-05T10:10:10.000000Z"},
{"parent_key": {"2022-09-05T10:10:10.000000Z": "2022-09-05T10:10:10.000Z"}},
[{"parent_key": "parent_key"}],
),
(None, {}, []),
],
)
def test_sub_slicer(last_record, expected, records):
date_time = "2022-09-05T10:10:10.000000Z"
parent_slicer = GreenHouseSlicer(cursor_field=date_time, options=None, request_cursor_field=None)
GreenHouseSlicer.read_records = MagicMock(return_value=records)
slicer = GreenHouseSubstreamSlicer(
cursor_field=date_time,
options=None,
request_cursor_field=None,
parent_stream=parent_slicer,
stream_slice_field=date_time,
parent_key="parent_key",
)
stream_slice = next(slicer.stream_slices(SyncMode, {})) if records else {}
slicer.update_cursor(stream_slice=stream_slice, last_record=last_record)
assert slicer.get_stream_state() == expected

View File

@@ -64,11 +64,12 @@ Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see
## Changelog
| Version | Date | Pull Request | Subject |
|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------|
| 0.2.9 | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime |
| 0.2.8 | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework |
| 0.2.7 | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users |
| 0.2.6 | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607) | Implement demographics streams support. Update SAT for demographics streams |
| 0.2.5 | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377) | Refactor the connector to use CDK. Implement additional stream support |
| Version | Date | Pull Request | Subject |
|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------|
| 0.2.10 | 2022-09-05 | [16338](https://github.com/airbytehq/airbyte/pull/16338) | Implement incremental syncs & fix SATs |
| 0.2.9 | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime |
| 0.2.8 | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework |
| 0.2.7 | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users |
| 0.2.6 | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607) | Implement demographics streams support. Update SAT for demographics streams |
| 0.2.5 | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377) | Refactor the connector to use CDK. Implement additional stream support |
| 0.2.4 | 2021-09-15 | [6238](https://github.com/airbytehq/airbyte/pull/6238) | Add identification of accessible streams for API keys with limited permissions |